From 745c7da0d8f5a298f3c9e9f60cc5709a4b8cb259 Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Sun, 16 Jul 2023 10:02:16 +0800 Subject: [PATCH 1/7] add digital signature --- .deploy_git | 2 +- db.json | 2 +- .../09/27/rust/rust-01-resource/index.html | 14 +- .../2022/10/04/rust/rust-02-basics/index.html | 14 +- .../10/11/rust/rust-03-ownership/index.html | 14 +- .../10/18/rust/rust-04-lifetime/index.html | 14 +- .../25/rust/rust-05-memory-layout/index.html | 14 +- .../30/rust/rust-06-smart-pointer/index.html | 14 +- .../code_analysis/geth.0.get.start/index.html | 14 +- .../rust-08-project-management/index.html | 14 +- .../geth/code_analysis/geth.1.rpc/index.html | 14 +- .../2022/11/13/rust/rust-cargo-doc/index.html | 14 +- public/2022/11/17/rust/rust-sugar/index.html | 14 +- public/2022/11/20/rust/rust-tools/index.html | 14 +- .../index.html | 14 +- public/2022/11/27/hello-world/index.html | 18 +- public/2022/11/28/eth-sign/index.html | 243 ------------- .../06/rust/rust-cargo-all-in-one/index.html | 18 +- .../rust-frequently-used-crates/index.html | 14 +- .../2022/12/28/rust/rust-07-macro/index.html | 14 +- public/2023/01/01/geth-fine-tune/index.html | 14 +- .../08/geth/code_analysis/geth.evm/index.html | 14 +- public/2023/01/13/rust/rust-async/index.html | 14 +- .../2023/01/15/blockchain.kademlia/index.html | 14 +- public/2023/01/22/MPT/index.html | 14 +- .../cryptography/two-party-ecdsa/index.html | 14 +- .../paillier-encryption/index.html | 14 +- .../2023/03/02/golang/go-reflect/index.html | 14 +- .../go-similar-concepts-comparison/index.html | 14 +- .../15/geth/tech_docs/geth.v1.10.0/index.html | 14 +- .../geth/tech_docs/geth.sync.mode/index.html | 14 +- .../25/geth/tech_docs/geth.prune/index.html | 14 +- .../04/01/rust/crates/rust-serde/index.html | 14 +- .../04/11/rust/rust-09-functional/index.html | 14 +- .../rust-std-data-structure-1/index.html | 14 +- .../rust-std-data-structure-2/index.html | 14 +- .../06/01/rust/rust-10-concurrency/index.html | 14 +- .../index.html | 14 +- .../index.html | 14 +- .../08/rust/rust_std/rust-std-sync/index.html | 18 +- .../cryptography-02-rsa}/index.html | 57 +-- .../index.html | 46 +-- .../index.html | 337 ++++++++++++++++++ .../zkp/zkp-a-brief-understanding/index.html | 33 +- public/archives/2022/09/index.html | 12 +- public/archives/2022/10/index.html | 12 +- public/archives/2022/11/index.html | 31 +- public/archives/2022/12/index.html | 12 +- public/archives/2022/index.html | 50 +-- public/archives/2022/page/2/index.html | 31 +- public/archives/2023/01/index.html | 12 +- public/archives/2023/02/index.html | 12 +- public/archives/2023/03/index.html | 12 +- public/archives/2023/04/index.html | 12 +- public/archives/2023/05/index.html | 12 +- public/archives/2023/06/index.html | 43 ++- public/archives/2023/index.html | 62 ++-- public/archives/2023/page/2/index.html | 50 +-- public/archives/2023/page/3/index.html | 31 +- public/archives/index.html | 62 ++-- public/archives/page/2/index.html | 50 +-- public/archives/page/3/index.html | 50 +-- public/archives/page/4/index.html | 12 +- public/archives/page/5/index.html | 12 +- .../elliptic_curve/point_addition.webp | Bin .../images/cryptography/rsa/rsa_signature.png | Bin 0 -> 283629 bytes public/index.html | 316 ++++++++++------ public/page/2/index.html | 149 ++++---- public/page/3/index.html | 140 +++++--- public/page/4/index.html | 32 +- public/page/5/index.html | 14 +- public/tags/blockchain/index.html | 12 +- public/tags/cargo/index.html | 12 +- public/tags/cryptography/index.html | 43 ++- public/tags/ecdsa/index.html | 12 +- public/tags/geth/index.html | 12 +- public/tags/golang/index.html | 12 +- public/tags/mpc/index.html | 12 +- public/tags/rust-crate/index.html | 12 +- public/tags/rust-std/index.html | 12 +- public/tags/rust/index.html | 12 +- public/tags/rust/page/2/index.html | 12 +- public/tags/zkp/index.html | 16 +- .../cryptography/cryptography-02-rsa.md | 114 ++++++ ...e.md => cryptography-03-elliptic-curve.md} | 6 +- .../cryptography/cryptography-03-rsa.md | 70 ---- .../cryptography-04-digital-signature.md | 134 +++++++ .../zkp/zkp-a-brief-understanding.md | 0 source/_posts/eth-sign.md | 9 - .../elliptic_curve/point_addition.webp | Bin .../images/cryptography/rsa/rsa_signature.png | Bin 0 -> 283629 bytes 91 files changed, 1703 insertions(+), 1290 deletions(-) delete mode 100644 public/2022/11/28/eth-sign/index.html rename public/2023/06/{17/cryptography/cryptography-03-rsa => 10/cryptography/cryptography-02-rsa}/index.html (65%) rename public/2023/06/{10/cryptography/cryptography-02-elliptic-curve => 17/cryptography/cryptography-03-elliptic-curve}/index.html (78%) create mode 100644 public/2023/06/20/cryptography/cryptography-04-digital-signature/index.html rename public/2023/06/20/{ => cryptography}/zkp/zkp-a-brief-understanding/index.html (80%) rename public/images/{ => cryptography}/elliptic_curve/point_addition.webp (100%) create mode 100644 public/images/cryptography/rsa/rsa_signature.png create mode 100644 source/_posts/cryptography/cryptography-02-rsa.md rename source/_posts/cryptography/{cryptography-02-elliptic-curve.md => cryptography-03-elliptic-curve.md} (96%) delete mode 100644 source/_posts/cryptography/cryptography-03-rsa.md create mode 100644 source/_posts/cryptography/cryptography-04-digital-signature.md rename source/_posts/{ => cryptography}/zkp/zkp-a-brief-understanding.md (100%) delete mode 100644 source/_posts/eth-sign.md rename source/images/{ => cryptography}/elliptic_curve/point_addition.webp (100%) create mode 100644 source/images/cryptography/rsa/rsa_signature.png diff --git a/.deploy_git b/.deploy_git index 18e00b60..1bfabe0a 160000 --- a/.deploy_git +++ b/.deploy_git @@ -1 +1 @@ -Subproject commit 18e00b6038e578ce86b8ab365b77eff9ca52d7d4 +Subproject commit 1bfabe0a4146de6ecd5c66f02f4dfd71040564c3 diff --git a/db.json b/db.json index 2c86c3c7..938931e2 100644 --- a/db.json +++ b/db.json @@ -1 +1 @@ -{"meta":{"version":1,"warehouse":"4.0.2"},"models":{"Asset":[{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.css","path":"fancybox/jquery.fancybox.min.css","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/js/jquery-3.4.1.min.js","path":"js/jquery-3.4.1.min.js","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/js/script.js","path":"js/script.js","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.js","path":"fancybox/jquery.fancybox.min.js","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/style.styl","path":"css/style.styl","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/FontAwesome.otf","path":"css/fonts/FontAwesome.otf","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.eot","path":"css/fonts/fontawesome-webfont.eot","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.svg","path":"css/fonts/fontawesome-webfont.svg","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.ttf","path":"css/fonts/fontawesome-webfont.ttf","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff","path":"css/fonts/fontawesome-webfont.woff","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff2","path":"css/fonts/fontawesome-webfont.woff2","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/images/banner.jpg","path":"css/images/banner.jpg","modified":1,"renderable":1},{"_id":"source/2017-552.pdf","path":"2017-552.pdf","modified":1,"renderable":0},{"_id":"source/images/evm.drawio.google.png","path":"images/evm.drawio.google.png","modified":1,"renderable":0},{"_id":"source/images/evm.layout.png","path":"images/evm.layout.png","modified":1,"renderable":0},{"_id":"source/images/geth_starts.drawio.png","path":"images/geth_starts.drawio.png","modified":1,"renderable":0},{"_id":"source/images/kademlia.locating.png","path":"images/kademlia.locating.png","modified":1,"renderable":0},{"_id":"source/images/kademlia.onlineprob.png","path":"images/kademlia.onlineprob.png","modified":1,"renderable":0},{"_id":"source/images/kademlia.subtree.png","path":"images/kademlia.subtree.png","modified":1,"renderable":0},{"_id":"source/images/mpt.png","path":"images/mpt.png","modified":1,"renderable":0},{"_id":"source/images/mpt.state.ref.png","path":"images/mpt.state.ref.png","modified":1,"renderable":0},{"_id":"source/images/trie.prefix.png","path":"images/trie.prefix.png","modified":1,"renderable":0},{"_id":"source/images/elliptic_curve/point_addition.webp","path":"images/elliptic_curve/point_addition.webp","modified":1,"renderable":0},{"_id":"source/images/geth/sync.mode.jpg","path":"images/geth/sync.mode.jpg","modified":1,"renderable":0},{"_id":"source/images/paillier/carmichael_thorem.png","path":"images/paillier/carmichael_thorem.png","modified":1,"renderable":0},{"_id":"source/images/paillier/carmichael_thorem_2.png","path":"images/paillier/carmichael_thorem_2.png","modified":1,"renderable":0},{"_id":"source/images/paillier/homomorphic_addition.png","path":"images/paillier/homomorphic_addition.png","modified":1,"renderable":0},{"_id":"source/images/paillier/homomorphic_mul.png","path":"images/paillier/homomorphic_mul.png","modified":1,"renderable":0},{"_id":"source/images/two_party_ecdsa/paillier_enc.png","path":"images/two_party_ecdsa/paillier_enc.png","modified":1,"renderable":0},{"_id":"source/images/two_party_ecdsa/schnorr_ecdsa_comparison.png","path":"images/two_party_ecdsa/schnorr_ecdsa_comparison.png","modified":1,"renderable":0},{"_id":"source/images/rust/macros/16.compile_process.png","path":"images/rust/macros/16.compile_process.png","modified":1,"renderable":0},{"_id":"source/images/rust/memory/trait_object_memory.png","path":"images/rust/memory/trait_object_memory.png","modified":1,"renderable":0},{"_id":"source/images/rust/ownership/move-string-2.png","path":"images/rust/ownership/move-string-2.png","modified":1,"renderable":0},{"_id":"source/images/rust/ownership/move-string.png","path":"images/rust/ownership/move-string.png","modified":1,"renderable":0},{"_id":"source/images/rust/pointers/cycle.ref.png","path":"images/rust/pointers/cycle.ref.png","modified":1,"renderable":0},{"_id":"source/images/rust/pointers/rc.png","path":"images/rust/pointers/rc.png","modified":1,"renderable":0}],"Cache":[{"_id":"source/images/evm.layout.png","hash":"6927cb3b922b2bcec926ec5d797e7ea8ea2b5d00","modified":1681440784560},{"_id":"source/images/kademlia.onlineprob.png","hash":"98c55aa819ef7047b3749c89eb4546f0d799a239","modified":1681443973575},{"_id":"source/_posts/blockchain.kademlia.md","hash":"0cb0522a114bc53d79f2db4a1bf2a02c29ef1c2f","modified":1681457596860},{"_id":"source/_posts/MPT.md","hash":"1d0394f11c3361b4474dbe49ced5c5751144fe91","modified":1681534320319},{"_id":"source/images/mpt.state.ref.png","hash":"ec747cf092820c0ab5f72d0f4f07f7705bb9ecf0","modified":1681533561017},{"_id":"source/images/elliptic_curve/point_addition.webp","hash":"ba6cd29aec4a694b53933d0824df180158f98316","modified":1689255155658},{"_id":"source/_posts/geth-fine-tune.md","hash":"5431cd654f7152fd5c590891a53568f54eec4125","modified":1682416094119},{"_id":"source/_posts/eth-sign.md","hash":"76a6376a62544d68438fb1759be07242a3e8e9ef","modified":1669607550181},{"_id":"source/images/geth/sync.mode.jpg","hash":"657cc148abc14f01e7483265c67936e73f79d0e4","modified":1687272599480},{"_id":"source/_posts/hello-world.md","hash":"8241e671f980a667f875b9a6493f17bd333a1cfb","modified":1680799419107},{"_id":"source/images/two_party_ecdsa/paillier_enc.png","hash":"132cdc74eb588951d1bd347b1b6e825912bc0460","modified":1682232953667},{"_id":"source/images/paillier/homomorphic_mul.png","hash":"e8a2420501140a8d48db202065aa221df92f19dc","modified":1682317735470},{"_id":"source/images/paillier/homomorphic_addition.png","hash":"26b0e4a070a7e29710e2ceace09bbdb79b3a883e","modified":1682317632123},{"_id":"source/_posts/golang/go-reflect.md","hash":"c2808bbd3bf422ab5679c246444975107b98fb1e","modified":1687070564968},{"_id":"source/_posts/rust/rust-01-resource.md","hash":"5e2c159128ccef35da0d3a2a72b6bb7e1c12fc73","modified":1688011565747},{"_id":"source/_posts/golang/go-similar-concepts-comparison.md","hash":"5de26ef1a5907db4264ff5e491b75aa2dfb43af1","modified":1687072042116},{"_id":"source/_posts/rust/rust-02-basics.md","hash":"be1111d868417c54b8b019938b8144e8be35df1f","modified":1682932033124},{"_id":"source/_posts/rust/rust-04-lifetime.md","hash":"d338498d7d159a3f94687082a10fc28ada706b16","modified":1682950129387},{"_id":"source/_posts/rust/rust-03-ownership.md","hash":"7d39498532088f438d76f1d0b42892bc985c0c95","modified":1682947052602},{"_id":"source/_posts/rust/rust-05-memory-layout.md","hash":"9a8c7dac3a23bca5f7692a3f6c0c8827dfd8c7e5","modified":1683082672719},{"_id":"source/_posts/rust/rust-07-macro.md","hash":"f6e51943ab413f5462baca1327c53ac20a8afcda","modified":1682950710350},{"_id":"source/_posts/rust/rust-06-smart-pointer.md","hash":"909ecf0ecf594651191e0953eca17aca7fa64669","modified":1683099103613},{"_id":"source/_posts/rust/rust-08-project-management.md","hash":"2c1ab1e7b8c8510935a694e33aa2c676437f0fd1","modified":1683099708892},{"_id":"source/_posts/rust/rust-async.md","hash":"f8b896f06752fcdb3c9fa34d2ed0ba958aef9c49","modified":1683106393753},{"_id":"source/_posts/rust/rust-10-concurrency.md","hash":"5bba6d7c1b0e3a24f07a4899f1162a93af8ad5b1","modified":1688283743897},{"_id":"source/_posts/rust/rust-09-functional.md","hash":"b2e1f9a46e02c5aa49ea19394e074777558a51eb","modified":1688003621688},{"_id":"source/_posts/rust/rust-cargo-doc.md","hash":"52303be5d9e342444a4cb661fa418ab68b28d640","modified":1683106131353},{"_id":"source/_posts/cryptography/cryptography-01-primitive-group-and-field.md","hash":"b109aef9a3c4ca55a7198c284cd007716b094044","modified":1689264071422},{"_id":"source/_posts/rust/rust-similar-concepts-comparison.md","hash":"17c7d67efd6315828a09762c633e7f8a0c9cdee2","modified":1688891607383},{"_id":"source/_posts/rust/rust-cargo-all-in-one.md","hash":"27eb587fe52f0461e1e30ed82b5d10a806e168f0","modified":1683108258682},{"_id":"source/_posts/rust/rust-sugar.md","hash":"dfa5a3f7bfd985118b97ae9055da496778b604e8","modified":1689060987147},{"_id":"source/_posts/cryptography/cryptography-03-rsa.md","hash":"00fd5cb12a4173ad1364e1c9d159fcf97e4040ad","modified":1689267250773},{"_id":"source/_posts/cryptography/cryptography-02-elliptic-curve.md","hash":"d3ef54a27b20eb8cc8db62c2fbfe632729b4c291","modified":1689264075193},{"_id":"source/_posts/rust/rust-tools.md","hash":"3019a116f156d05a95fd8fc76fa8b1380f778f24","modified":1683105904042},{"_id":"source/_posts/cryptography/paillier-encryption.md","hash":"4e43aa2d126bcb334563262563071c764c3ae19f","modified":1682317904177},{"_id":"source/_posts/cryptography/two-party-ecdsa.md","hash":"e94506f2f21233958c8f48184fad671919f32f8b","modified":1682317913595},{"_id":"source/_posts/zkp/zkp-a-brief-understanding.md","hash":"2e0f55f389b56b49dea2418cf062722ca339a0ea","modified":1689173902472},{"_id":"source/images/rust/memory/trait_object_memory.png","hash":"dd750cffa7bca5bde7856cfb65f4500de286c96f","modified":1683082638012},{"_id":"source/images/rust/ownership/move-string.png","hash":"d708edd6fb84b9d4ebe75556932f6b920934ca9a","modified":1682934539699},{"_id":"source/images/rust/pointers/rc.png","hash":"f568f69808ef0357ea5e6d42667d863b1139cbf1","modified":1683085079705},{"_id":"source/_posts/geth/tech_docs/geth.v1.10.0.md","hash":"d303922d31b987d5b57080fb6833b84e568de342","modified":1687100789245},{"_id":"source/_posts/geth/tech_docs/geth.prune.md","hash":"9ab8f315c3374f67ff7ac6f74a7fbef0ca9f7894","modified":1687341103628},{"_id":"source/_posts/geth/tech_docs/geth.sync.mode.md","hash":"7502b826b5ecbdd02f759a1780528dae344ccad7","modified":1687309420833},{"_id":"source/_posts/rust/crates/rust-serde.md","hash":"9b985750a751a7bd28effe9e97147e4207a5f093","modified":1688053726930},{"_id":"source/_posts/rust/crates/rust-frequently-used-crates.md","hash":"39aec8a77f62d6c3b164ecef0663a612423afd63","modified":1683106159269},{"_id":"source/_posts/rust/rust_std/rust-std-data-structure-1.md","hash":"34627ff9cff48a6a79a36d4e88cc4d32e118c3ae","modified":1689070720048},{"_id":"source/_posts/rust/rust_std/rust-smart-pointer-and-internal-mutibility.md","hash":"d97435f2c35ffcd4feb8a1aff787c7c20922438a","modified":1688915715385},{"_id":"source/_posts/geth/code_analysis/geth.1.rpc.md","hash":"623aec4f3cd8afb85b7b30d8bfde50bb2e5a4d4a","modified":1682614464069},{"_id":"source/_posts/rust/rust_std/rust-std-data-structure-2.md","hash":"32b80b9540433ffa77a3a7969e06921eb9ddbbca","modified":1688564475719},{"_id":"source/_posts/rust/rust_std/rust-std-sync.md","hash":"bf648427835ab0d834b2701b28bdfd35088aeb2e","modified":1688566866619},{"_id":"source/_posts/geth/code_analysis/geth.evm.md","hash":"ecea3e42003911dd58a2d6a9b8293f02ccb16a75","modified":1681441326144},{"_id":"source/_posts/geth/code_analysis/geth.0.get.start.md","hash":"6cd5256bae5e43f3dfcd341e27c52eccf8848f60","modified":1682434784499},{"_id":"source/images/evm.drawio.google.png","hash":"413a44d66153c93e2c31172d988e051a2bed9940","modified":1681436195264},{"_id":"source/images/geth_starts.drawio.png","hash":"3c426bd608121669f232aaa1b05ed7a342287fc8","modified":1682005494018},{"_id":"source/images/kademlia.subtree.png","hash":"d3198467460972151f4b2fe2b56fab0dd9e411ca","modified":1681442854122},{"_id":"source/images/trie.prefix.png","hash":"67d26d416faf0cc43ee35b4acda598e88dad2949","modified":1681520956971},{"_id":"source/images/paillier/carmichael_thorem.png","hash":"75b4d32eb2233db653bca52cdea4a624555b5ce4","modified":1682316092261},{"_id":"source/images/paillier/carmichael_thorem_2.png","hash":"29b103c94c36f4520ca8b675486af3d914998ac1","modified":1682316415073},{"_id":"source/images/two_party_ecdsa/schnorr_ecdsa_comparison.png","hash":"69cbed302af467a4e99653dfb51dca45c4a5a6f3","modified":1682231680569},{"_id":"source/images/rust/ownership/move-string-2.png","hash":"83342aed7ee254c099ada5d49ed8b9ba52a451a0","modified":1682934544128},{"_id":"source/images/rust/pointers/cycle.ref.png","hash":"ed99e2c6020833ccd5af751cf6eb2031cf47d9aa","modified":1683098263386},{"_id":"source/images/kademlia.locating.png","hash":"702d6b779294a3c6e033cc9bde14ef8950982310","modified":1681455579355},{"_id":"source/images/rust/macros/16.compile_process.png","hash":"444080319ca2101672941346988f78ed9a37c32d","modified":1682908885234},{"_id":"source/images/mpt.png","hash":"700032035bdcd793f94da522330552727b00e5a3","modified":1681533347412},{"_id":"node_modules/hexo-theme-landscape/package.json","hash":"9a94875cbf4c27fbe2e63da0496242addc6d2876","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/LICENSE","hash":"c480fce396b23997ee23cc535518ffaaf7f458f8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/README.md","hash":"d2772ece6d4422ccdaa0359c3e07588834044052","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/_config.yml","hash":"b608c1f1322760dce9805285a602a95832730a2e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/de.yml","hash":"3ebf0775abbee928c8d7bda943c191d166ded0d3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/en.yml","hash":"3083f319b352d21d80fc5e20113ddf27889c9d11","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/es.yml","hash":"76edb1171b86532ef12cfd15f5f2c1ac3949f061","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/fr.yml","hash":"415e1c580ced8e4ce20b3b0aeedc3610341c76fb","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/hu.yml","hash":"284d557130bf54a74e7dcef9d42096130e4d9550","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/it.yml","hash":"89b7d91306b2c1a0f3ac023b657bf974f798a1e8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ja.yml","hash":"a73e1b9c80fd6e930e2628b393bfe3fb716a21a9","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/pt.yml","hash":"57d07b75d434fbfc33b0ddb543021cb5f53318a8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/mn.yml","hash":"2e7523951072a9403ead3840ad823edd1084c116","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/no.yml","hash":"965a171e70347215ec726952e63f5b47930931ef","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ko.yml","hash":"881d6a0a101706e0452af81c580218e0bfddd9cf","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/zh-CN.yml","hash":"1efd95774f401c80193eac6ee3f1794bfe93dc5a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ru.yml","hash":"4fda301bbd8b39f2c714e2c934eccc4b27c0a2b0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/nl.yml","hash":"12ed59faba1fc4e8cdd1d42ab55ef518dde8039c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/category.ejs","hash":"765426a9c8236828dc34759e604cc2c52292835a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/archive.ejs","hash":"2703b07cc8ac64ae46d1d263f4653013c7e1666b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/tr.yml","hash":"a1cdbfa17682d7a971de8ab8588bf57c74224b5b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/index.ejs","hash":"aa1b4456907bdb43e629be3931547e2d29ac58c8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/zh-TW.yml","hash":"53ce3000c5f767759c7d2c4efcaa9049788599c3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/page.ejs","hash":"7d80e4e36b14d30a7cd2ac1f61376d9ebf264e8b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/layout.ejs","hash":"0d1765036e4874500e68256fedb7470e96eeb6ee","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/scripts/fancybox.js","hash":"c857d7a5e4a5d71c743a009c5932bf84229db428","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/tag.ejs","hash":"eaa7b4ccb2ca7befb90142e4e68995fb1ea68b2e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/post.ejs","hash":"7d80e4e36b14d30a7cd2ac1f61376d9ebf264e8b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/after-footer.ejs","hash":"414914ebb159fac1922b056b905e570ac7521925","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/archive-post.ejs","hash":"c7a71425a946d05414c069ec91811b5c09a92c47","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/archive.ejs","hash":"7cb70a7a54f8c7ae49b10d1f37c0a9b74eab8826","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/footer.ejs","hash":"3656eb692254346671abc03cb3ba1459829e0dce","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/article.ejs","hash":"dfd555c00e85ffc4207c88968d12b219c1f086ec","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/google-analytics.ejs","hash":"2ea7442ea1e1a8ab4e41e26c563f58413b59a3d0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/archive.ejs","hash":"beb4a86fcc82a9bdda9289b59db5a1988918bec3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/gauges-analytics.ejs","hash":"21a1e2a3907d1a3dad1cd0ab855fe6735f233c74","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/mobile-nav.ejs","hash":"e952a532dfc583930a666b9d4479c32d4a84b44e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/head.ejs","hash":"f215d92a882247a7cc5ea80b241bedfcec0ea6ca","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/sidebar.ejs","hash":"930da35cc2d447a92e5ee8f835735e6fd2232469","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/tag.ejs","hash":"2de380865df9ab5f577f7d3bcadf44261eb5faae","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/header.ejs","hash":"c1acd247e14588cdf101a69460cb8319c18cd078","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/category.ejs","hash":"dd1e5af3c6af3f5d6c85dfd5ca1766faed6a0b05","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/recent_posts.ejs","hash":"60c4b012dcc656438ff59997e60367e5a21ab746","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/tagcloud.ejs","hash":"b4a2079101643f63993dcdb32925c9b071763b46","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/js/script.js","hash":"998ed4c5b147e1299bf62beebf33514474f28112","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_variables.styl","hash":"581b0cbefdaa5f894922133989dd2d3bf71ded79","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/style.styl","hash":"9c451e5efd72c5bb8b56e8c2b94be731e99db05b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.css","hash":"1be9b79be02a1cfc5d96c4a5e0feb8f472babd95","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_extend.styl","hash":"222fbe6d222531d61c1ef0f868c90f747b1c2ced","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/date.ejs","hash":"f1458584b679545830b75bef2526e2f3eb931045","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/gallery.ejs","hash":"3d9d81a3c693ff2378ef06ddb6810254e509de5b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/nav.ejs","hash":"16a904de7bceccbb36b4267565f2215704db2880","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/tag.ejs","hash":"2fcb0bf9c8847a644167a27824c9bb19ac74dd14","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/title.ejs","hash":"4d7e62574ddf46de9b41605fe3140d77b5ddb26d","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/archive.styl","hash":"db15f5677dc68f1730e82190bab69c24611ca292","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/article.styl","hash":"80759482d07063c091e940f964a1cf6693d3d406","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/footer.styl","hash":"e35a060b8512031048919709a8e7b1ec0e40bc1b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/comment.styl","hash":"79d280d8d203abb3bd933ca9b8e38c78ec684987","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/category.ejs","hash":"c6bcd0e04271ffca81da25bcff5adf3d46f02fc0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/header.styl","hash":"85ab11e082f4dd86dde72bed653d57ec5381f30c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/highlight.styl","hash":"bf4e7be1968dad495b04e83c95eac14c4d0ad7c0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/mobile.styl","hash":"a399cf9e1e1cec3e4269066e2948d7ae5854d745","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar-aside.styl","hash":"890349df5145abf46ce7712010c89237900b3713","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar.styl","hash":"404ec059dc674a48b9ab89cd83f258dec4dcb24d","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar-bottom.styl","hash":"8fd4f30d319542babfd31f087ddbac550f000a8a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_util/grid.styl","hash":"0bf55ee5d09f193e249083602ac5fcdb1e571aed","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_util/mixin.styl","hash":"44f32767d9fd3c1c08a60d91f181ee53c8f0dbb3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.js","hash":"6181412e73966696d08e1e5b1243a572d0f22ba6","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/js/jquery-3.4.1.min.js","hash":"88523924351bac0b5d560fe0c5781e2556e7693d","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff2","hash":"d6f48cba7d076fb6f2fd6ba993a75b9dc1ecbf0c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff","hash":"28b782240b3e76db824e12c02754a9731a167527","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.eot","hash":"d980c2ce873dc43af460d4d572d441304499f400","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/FontAwesome.otf","hash":"048707bc52ac4b6563aaa383bfe8660a0ddc908c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.ttf","hash":"13b1eab65a983c7a73bc7997c479d66943f7c6cb","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/images/banner.jpg","hash":"f44aa591089fcb3ec79770a1e102fd3289a7c6a6","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.svg","hash":"98a8aa5cf7d62c2eff5f07ede8d844b874ef06ed","modified":1669606834570},{"_id":"source/2017-552.pdf","hash":"b1a08857c1f6532f1fbb718c48a34fd48ea7da70","modified":1682236639612},{"_id":"public/2023/06/20/zkp/zkp-a-brief-understanding/index.html","hash":"2932d6f4e874456026b3ee856550c9ff41533dc1","modified":1689267302416},{"_id":"public/2023/06/17/cryptography/cryptography-03-rsa/index.html","hash":"1da046d8ea0bb8db506868ec1b165b5739e75f0d","modified":1689267302416},{"_id":"public/2023/06/10/cryptography/cryptography-02-elliptic-curve/index.html","hash":"2ebc1d33c8cfede924e6e841f6ce41f503ac6612","modified":1689267302416},{"_id":"public/2023/06/01/rust/rust-10-concurrency/index.html","hash":"4d0fa39cd6d7c76922c5f4b9a179c1701b279513","modified":1689267302416},{"_id":"public/2023/03/25/geth/tech_docs/geth.prune/index.html","hash":"30599e6334ac9779c1d1c1e063c6272d92a01d60","modified":1689267302416},{"_id":"public/2023/04/01/rust/crates/rust-serde/index.html","hash":"b6d8edf99edb4d82f32ba4fa38801437105ec323","modified":1689267302416},{"_id":"public/2023/04/11/rust/rust-09-functional/index.html","hash":"f59beb2a9ae7c3227c1e33d8ed51e90b00edac6c","modified":1689267302416},{"_id":"public/2023/03/05/golang/go-similar-concepts-comparison/index.html","hash":"7ef4625d43e210240df45cbb2eca79337da6a399","modified":1689267302416},{"_id":"public/2023/02/07/cryptography/two-party-ecdsa/index.html","hash":"8b207984b06790882610c780d4309853f28a3f71","modified":1689267302416},{"_id":"public/2023/01/22/MPT/index.html","hash":"24a154eafc38313aa5c07e5c32ba9d7200c267ad","modified":1689267302416},{"_id":"public/2023/01/01/geth-fine-tune/index.html","hash":"b15f261756039fb3f1fe3bdefc988da4e8300d49","modified":1689267302416},{"_id":"public/2022/12/13/rust/crates/rust-frequently-used-crates/index.html","hash":"97a126f4e0931cd1127e7ddd100d53db9a71082c","modified":1689267302416},{"_id":"public/2022/11/28/eth-sign/index.html","hash":"718b4103a7ccf6b8a33700d6efc51a476d4a70d2","modified":1689267302416},{"_id":"public/2022/11/27/hello-world/index.html","hash":"80ed788a3a3ad1f8926be8924849ae5f22c522fb","modified":1689267302416},{"_id":"public/2022/11/20/rust/rust-tools/index.html","hash":"8011dfeb47ba5e84e00e485ba1f59586b1f405c8","modified":1689267302416},{"_id":"public/2022/11/13/rust/rust-cargo-doc/index.html","hash":"19298b1d7c59513cda44e9ee0fe8c701f4f0d581","modified":1689267302416},{"_id":"public/2022/11/08/geth/code_analysis/geth.1.rpc/index.html","hash":"c9f4ef6eac7038be7fbe321c02d3c206150d19de","modified":1689267302416},{"_id":"public/2022/10/25/rust/rust-05-memory-layout/index.html","hash":"d083eae7c0f10616b944837daaba85b2ca8e63e3","modified":1689267302416},{"_id":"public/2022/09/27/rust/rust-01-resource/index.html","hash":"4a5ad9e72e0df364ed658735c63a0467b2e274b1","modified":1689267302416},{"_id":"public/archives/index.html","hash":"b26f6e0dd685861e45389b5ae789b489306f324d","modified":1689267302416},{"_id":"public/archives/page/2/index.html","hash":"13bf2a1171af57af2b910fa745e14dc94e1cc7ac","modified":1689267302416},{"_id":"public/archives/page/3/index.html","hash":"9c062822896543a5e2df0ffed5afbcaac1f19620","modified":1689267302416},{"_id":"public/archives/page/4/index.html","hash":"615d799dd9cdd6a89fcd1aa801044fea99d0f4e1","modified":1689267302416},{"_id":"public/archives/page/5/index.html","hash":"68088e06cebf1cd86edb2dcc3da4f0f3a79abeca","modified":1689267302416},{"_id":"public/archives/2022/index.html","hash":"e1b68ff1386b22f992ecb792512bce0c81471fda","modified":1689267302416},{"_id":"public/archives/2022/09/index.html","hash":"2d9dec992d9793d7faf92331ced7e2c0893fc628","modified":1689267302416},{"_id":"public/archives/2022/page/2/index.html","hash":"d3b4d22a0df7511c590065832bbdb83a27626412","modified":1689267302416},{"_id":"public/archives/2022/11/index.html","hash":"2d86c51cf036245c0444508ff9078d0ab3ae7b0c","modified":1689267302416},{"_id":"public/archives/2022/10/index.html","hash":"0db4f58ca82e62e5e41109bb3995989407ebee2f","modified":1689267302416},{"_id":"public/archives/2022/12/index.html","hash":"e8973cc3006651081b8165ca03c344353c121308","modified":1689267302416},{"_id":"public/archives/2023/index.html","hash":"bfcf7d875f1db873fffffb35bf469bf7faa04337","modified":1689267302416},{"_id":"public/archives/2023/page/2/index.html","hash":"3378570ac96475bba027ffe2b5bcd7085eab5f4f","modified":1689267302416},{"_id":"public/archives/2023/page/3/index.html","hash":"f690fc3bfd31262ed096434c3c5209acdb75a517","modified":1689267302416},{"_id":"public/archives/2023/01/index.html","hash":"f2e5aae467c44820380e4013062de6200af91713","modified":1689267302416},{"_id":"public/archives/2023/02/index.html","hash":"97350905e138f0a39165abb99dae5c6aa766ebd7","modified":1689267302416},{"_id":"public/archives/2023/03/index.html","hash":"710cd060c7669a22bc36e19914933c5cec1e4d52","modified":1689267302416},{"_id":"public/archives/2023/04/index.html","hash":"f94514e2a73ebcd34eb2de4dae42768391617754","modified":1689267302416},{"_id":"public/archives/2023/05/index.html","hash":"d02d80c39e5b2cd50ad3609a277107ebc3025cc8","modified":1689267302416},{"_id":"public/archives/2023/06/index.html","hash":"606db95140374582e47446353fd6af27593f545e","modified":1689267302416},{"_id":"public/tags/blockchain/index.html","hash":"bb3e39ada873831c70a9f7ae6562a8c2cd0f5268","modified":1689267302416},{"_id":"public/tags/geth/index.html","hash":"23f6e67c14ecb91130841423774e08d08ce5955a","modified":1689267302416},{"_id":"public/tags/golang/index.html","hash":"da06c8f9f2c5e476474ed0de2c2d9f300ee75316","modified":1689267302416},{"_id":"public/tags/rust/index.html","hash":"e3133a5841f411d290f211e2119a6e7feae5f1f9","modified":1689267302416},{"_id":"public/tags/cargo/index.html","hash":"5e4eb2773b0e9b130b8bfc1b357a4443c99a6515","modified":1689267302416},{"_id":"public/tags/cryptography/index.html","hash":"f5c48068d7d3aeacdb2a50411462a75d40876bd7","modified":1689267302416},{"_id":"public/tags/zkp/index.html","hash":"b4dcf0f3b03d19763f83e9b5d053060e3b20c1d2","modified":1689267302416},{"_id":"public/tags/rust/page/2/index.html","hash":"c8f0911a1188a3fd24c477c63d3445feca73fef8","modified":1689267302416},{"_id":"public/tags/mpc/index.html","hash":"c53ecbdeb3ed9e736201c022ee63955dd90a1d0c","modified":1689267302416},{"_id":"public/tags/ecdsa/index.html","hash":"2119718393b2a77da4f95d7080a42a1b75ce93f7","modified":1689267302416},{"_id":"public/tags/rust-crate/index.html","hash":"e35ebad0ea539f26f07bf04bbdae2cbd6d6c77d9","modified":1689267302416},{"_id":"public/tags/rust-std/index.html","hash":"cdafc1b79d3b3d019e565bccf8705b5af02db2a8","modified":1689267302416},{"_id":"public/2023/06/08/rust/rust_std/rust-std-sync/index.html","hash":"058e8990c4467d86c835419ecf78796ced310e80","modified":1689267302416},{"_id":"public/2023/06/03/rust/rust_std/rust-smart-pointer-and-internal-mutibility/index.html","hash":"735fbfd942cf3e9dad9150be1c5227d91f28a561","modified":1689267302416},{"_id":"public/2023/06/03/cryptography/cryptography-01-primitive-group-and-field/index.html","hash":"d20bc6151690c8b56348795e757507e446a91e78","modified":1689267302416},{"_id":"public/2023/05/02/rust/rust_std/rust-std-data-structure-2/index.html","hash":"9bcc9796d999beba7164f419acb6a80a0f6ca326","modified":1689267302416},{"_id":"public/2023/05/01/rust/rust_std/rust-std-data-structure-1/index.html","hash":"c5378f2f1c7b6d18c6452e2cef85eea7d9b05036","modified":1689267302416},{"_id":"public/2023/03/15/geth/tech_docs/geth.v1.10.0/index.html","hash":"9a722579070e4e75ff6b498f8eae59e7290940dc","modified":1689267302416},{"_id":"public/2023/03/18/geth/tech_docs/geth.sync.mode/index.html","hash":"92d962427bc5b8b15494734d442bd52b44f01c1a","modified":1689267302416},{"_id":"public/2023/02/23/cryptography/paillier-encryption/index.html","hash":"e54db8f7129c2ae6917ed1ad1e9b86c4886245ce","modified":1689267302416},{"_id":"public/2023/03/02/golang/go-reflect/index.html","hash":"ad570826baa973bd8e924f328a06c2a885b5de4b","modified":1689267302416},{"_id":"public/2023/01/15/blockchain.kademlia/index.html","hash":"d7a6b3242bfadf95bf2f23c494a95abf1fee89cc","modified":1689267302416},{"_id":"public/2023/01/13/rust/rust-async/index.html","hash":"fd9cf2ce0eb65104c85a35811ec9fcb415a89949","modified":1689267302416},{"_id":"public/2023/01/08/geth/code_analysis/geth.evm/index.html","hash":"40a3c03dd9e3e6998915d4c69b0cd73bb9435940","modified":1689267302416},{"_id":"public/2022/12/28/rust/rust-07-macro/index.html","hash":"b9295db63e6b933be9ae1cc3b6333ed7d0f8c8c6","modified":1689267302416},{"_id":"public/2022/12/06/rust/rust-cargo-all-in-one/index.html","hash":"54dd77b0147a3c6f5c921515576348e34598708a","modified":1689267302416},{"_id":"public/2022/11/23/rust/rust-similar-concepts-comparison/index.html","hash":"04d891c41303f8ab8697ab56037329a30a3eeb96","modified":1689267302416},{"_id":"public/2022/11/17/rust/rust-sugar/index.html","hash":"e43566fece3f22a33312abd5f74e8ae40e0e928c","modified":1689267302416},{"_id":"public/2022/11/01/geth/code_analysis/geth.0.get.start/index.html","hash":"295c2545973f3aef6ddaf3ccb6fd396827abc2d0","modified":1689267302416},{"_id":"public/2022/11/06/rust/rust-08-project-management/index.html","hash":"e12e0a4ac72beb800e2309b6914e43386d123fee","modified":1689267302416},{"_id":"public/2022/10/30/rust/rust-06-smart-pointer/index.html","hash":"96da636bb5aad209d09e7ce12feb4cf50ca68e6d","modified":1689267302416},{"_id":"public/2022/10/18/rust/rust-04-lifetime/index.html","hash":"0dba0b342c9ca2b5dae764aa85049d2f6d4ed032","modified":1689267302416},{"_id":"public/2022/10/11/rust/rust-03-ownership/index.html","hash":"ced074bffa6ad877ee9566d8f0ed6758f3a4fee8","modified":1689267302416},{"_id":"public/2022/10/04/rust/rust-02-basics/index.html","hash":"6055b1f52ccd9f9b2f70f804e993c4e02c78819f","modified":1689267302416},{"_id":"public/index.html","hash":"47873e579076107b0e1bba21d1726ba73467bba1","modified":1689267302416},{"_id":"public/page/2/index.html","hash":"af0d327142257142c89da14c929faea639f5d2bd","modified":1689267302416},{"_id":"public/page/5/index.html","hash":"73fe79c7012b1a8c7c1f3cf37d0b4402ec9f811a","modified":1689267302416},{"_id":"public/page/3/index.html","hash":"4c6007d265222266143cdaebe0b5527d6578b818","modified":1689267302416},{"_id":"public/page/4/index.html","hash":"07303887258ded22a41b87ee84b39b4f8ce5c886","modified":1689267302416},{"_id":"public/images/evm.layout.png","hash":"6927cb3b922b2bcec926ec5d797e7ea8ea2b5d00","modified":1689267302416},{"_id":"public/images/kademlia.onlineprob.png","hash":"98c55aa819ef7047b3749c89eb4546f0d799a239","modified":1689267302416},{"_id":"public/images/elliptic_curve/point_addition.webp","hash":"ba6cd29aec4a694b53933d0824df180158f98316","modified":1689267302416},{"_id":"public/images/mpt.state.ref.png","hash":"ec747cf092820c0ab5f72d0f4f07f7705bb9ecf0","modified":1689267302416},{"_id":"public/images/geth/sync.mode.jpg","hash":"657cc148abc14f01e7483265c67936e73f79d0e4","modified":1689267302416},{"_id":"public/images/paillier/homomorphic_addition.png","hash":"26b0e4a070a7e29710e2ceace09bbdb79b3a883e","modified":1689267302416},{"_id":"public/images/two_party_ecdsa/paillier_enc.png","hash":"132cdc74eb588951d1bd347b1b6e825912bc0460","modified":1689267302416},{"_id":"public/images/paillier/homomorphic_mul.png","hash":"e8a2420501140a8d48db202065aa221df92f19dc","modified":1689267302416},{"_id":"public/images/rust/memory/trait_object_memory.png","hash":"dd750cffa7bca5bde7856cfb65f4500de286c96f","modified":1689267302416},{"_id":"public/images/rust/ownership/move-string.png","hash":"d708edd6fb84b9d4ebe75556932f6b920934ca9a","modified":1689267302416},{"_id":"public/images/rust/pointers/rc.png","hash":"f568f69808ef0357ea5e6d42667d863b1139cbf1","modified":1689267302416},{"_id":"public/fancybox/jquery.fancybox.min.css","hash":"1be9b79be02a1cfc5d96c4a5e0feb8f472babd95","modified":1689267302416},{"_id":"public/js/script.js","hash":"998ed4c5b147e1299bf62beebf33514474f28112","modified":1689267302416},{"_id":"public/js/jquery-3.4.1.min.js","hash":"88523924351bac0b5d560fe0c5781e2556e7693d","modified":1689267302416},{"_id":"public/css/style.css","hash":"4da345d832a2682bcaee3ab3e22c15e3cd0e9cde","modified":1689267302416},{"_id":"public/fancybox/jquery.fancybox.min.js","hash":"6181412e73966696d08e1e5b1243a572d0f22ba6","modified":1689267302416},{"_id":"public/css/fonts/fontawesome-webfont.woff2","hash":"d6f48cba7d076fb6f2fd6ba993a75b9dc1ecbf0c","modified":1689267302416},{"_id":"public/images/geth_starts.drawio.png","hash":"3c426bd608121669f232aaa1b05ed7a342287fc8","modified":1689267302416},{"_id":"public/images/kademlia.subtree.png","hash":"d3198467460972151f4b2fe2b56fab0dd9e411ca","modified":1689267302416},{"_id":"public/css/fonts/fontawesome-webfont.woff","hash":"28b782240b3e76db824e12c02754a9731a167527","modified":1689267302416},{"_id":"public/images/trie.prefix.png","hash":"67d26d416faf0cc43ee35b4acda598e88dad2949","modified":1689267302416},{"_id":"public/images/paillier/carmichael_thorem.png","hash":"75b4d32eb2233db653bca52cdea4a624555b5ce4","modified":1689267302416},{"_id":"public/images/paillier/carmichael_thorem_2.png","hash":"29b103c94c36f4520ca8b675486af3d914998ac1","modified":1689267302416},{"_id":"public/images/evm.drawio.google.png","hash":"413a44d66153c93e2c31172d988e051a2bed9940","modified":1689267302416},{"_id":"public/images/two_party_ecdsa/schnorr_ecdsa_comparison.png","hash":"69cbed302af467a4e99653dfb51dca45c4a5a6f3","modified":1689267302416},{"_id":"public/images/rust/pointers/cycle.ref.png","hash":"ed99e2c6020833ccd5af751cf6eb2031cf47d9aa","modified":1689267302416},{"_id":"public/images/rust/ownership/move-string-2.png","hash":"83342aed7ee254c099ada5d49ed8b9ba52a451a0","modified":1689267302416},{"_id":"public/css/fonts/fontawesome-webfont.eot","hash":"d980c2ce873dc43af460d4d572d441304499f400","modified":1689267302416},{"_id":"public/css/fonts/FontAwesome.otf","hash":"048707bc52ac4b6563aaa383bfe8660a0ddc908c","modified":1689267302416},{"_id":"public/css/fonts/fontawesome-webfont.ttf","hash":"13b1eab65a983c7a73bc7997c479d66943f7c6cb","modified":1689267302416},{"_id":"public/css/images/banner.jpg","hash":"f44aa591089fcb3ec79770a1e102fd3289a7c6a6","modified":1689267302416},{"_id":"public/images/kademlia.locating.png","hash":"702d6b779294a3c6e033cc9bde14ef8950982310","modified":1689267302416},{"_id":"public/images/rust/macros/16.compile_process.png","hash":"444080319ca2101672941346988f78ed9a37c32d","modified":1689267302416},{"_id":"public/images/mpt.png","hash":"700032035bdcd793f94da522330552727b00e5a3","modified":1689267302416},{"_id":"public/css/fonts/fontawesome-webfont.svg","hash":"98a8aa5cf7d62c2eff5f07ede8d844b874ef06ed","modified":1689267302416},{"_id":"public/2017-552.pdf","hash":"b1a08857c1f6532f1fbb718c48a34fd48ea7da70","modified":1689267302416}],"Category":[],"Data":[],"Page":[],"Post":[{"title":"MPT","date":"2023-01-22T09:25:36.000Z","_content":"\n# trie\na trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.\n\n![prefix trie](/images/trie.prefix.png)\nIn general, the nodes of a Trie look like this:\n```\n[ [Ia, Ib, … I*], value]\n```\n[Ia, Ib, ... I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value\n\n# MPT\nEach block of Ethereum contains three MPT trees, respectively\n\n- Transaction tree\n- Receipt tree\n- State tree\n\nIn the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is\n\n![state reference](/images/mpt.state.ref.png)\n\n- use []byte as key, other than string\n- nibble: the smallest unit of the key type (4 bit)\n- Use hashes to refer to nodes instead of memory pointers\n\nthere are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows\n```\nfullNode: [i0, i1, i2, … i15, hash] \nshortNode： [ [k0, k1, … kn], hash ] // first element is an array\n```\nif the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.\n\n![mpt](/images/mpt.png)\n\nUse the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value\n|hex char| bits | pointing to | odd/even | 2nd niddle padding |\n| -----|:----:|:----:|:----:|-------|\n|0| 0000 | node | even | no |\n|1| 0001 | node | odd | yes |\n|2| 0010 | value | even | no |\n|3| 0011 | value | odd | yes |\n\nthis encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type\n\nIn the trie module, there is a `Database` object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database\n\n# reference\n- [github](https://github.com/agiletechvn/go-ethereum-code-analysis/blob/master/trie-analysis.md)\n- [yangzhe's blog](http://yangzhe.me/2019/01/12/ethereum-trie-part-1/)\n- [yangzhe's blod](http://yangzhe.me/2019/01/18/ethereum-trie-part-2/)","source":"_posts/MPT.md","raw":"---\ntitle: MPT\ndate: 2023-01-22 17:25:36\ntags: [blockchain, geth]\n---\n\n# trie\na trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.\n\n![prefix trie](/images/trie.prefix.png)\nIn general, the nodes of a Trie look like this:\n```\n[ [Ia, Ib, … I*], value]\n```\n[Ia, Ib, ... I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value\n\n# MPT\nEach block of Ethereum contains three MPT trees, respectively\n\n- Transaction tree\n- Receipt tree\n- State tree\n\nIn the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is\n\n![state reference](/images/mpt.state.ref.png)\n\n- use []byte as key, other than string\n- nibble: the smallest unit of the key type (4 bit)\n- Use hashes to refer to nodes instead of memory pointers\n\nthere are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows\n```\nfullNode: [i0, i1, i2, … i15, hash] \nshortNode： [ [k0, k1, … kn], hash ] // first element is an array\n```\nif the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.\n\n![mpt](/images/mpt.png)\n\nUse the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value\n|hex char| bits | pointing to | odd/even | 2nd niddle padding |\n| -----|:----:|:----:|:----:|-------|\n|0| 0000 | node | even | no |\n|1| 0001 | node | odd | yes |\n|2| 0010 | value | even | no |\n|3| 0011 | value | odd | yes |\n\nthis encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type\n\nIn the trie module, there is a `Database` object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database\n\n# reference\n- [github](https://github.com/agiletechvn/go-ethereum-code-analysis/blob/master/trie-analysis.md)\n- [yangzhe's blog](http://yangzhe.me/2019/01/12/ethereum-trie-part-1/)\n- [yangzhe's blod](http://yangzhe.me/2019/01/18/ethereum-trie-part-2/)","slug":"MPT","published":1,"updated":"2023-04-15T04:52:00.319Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpi00006hsj5wv8brh1","content":"

trie

a trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.

\n

\"prefix
In general, the nodes of a Trie look like this:

\n
1
[ [Ia, Ib, … I*], value]
\n

[Ia, Ib, … I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value

\n

MPT

Each block of Ethereum contains three MPT trees, respectively

\n\n

In the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is

\n

\"state

\n\n

there are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows

\n
1
2
fullNode: [i0, i1, i2, … i15, hash]  
shortNode： [ [k0, k1, … kn], hash ] // first element is an array
\n

if the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.

\n

\"mpt\"

\n

Use the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
hex charbitspointing toodd/even2nd niddle padding
00000nodeevenno
10001nodeoddyes
20010valueevenno
30011valueoddyes
\n

this encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type

\n

In the trie module, there is a Database object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database

\n

reference

\n","site":{"data":{}},"excerpt":"","more":"

trie

a trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.

\n

\"prefix
In general, the nodes of a Trie look like this:

\n
1
[ [Ia, Ib, … I*], value]
\n

[Ia, Ib, … I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value

\n

MPT

Each block of Ethereum contains three MPT trees, respectively

\n\n

In the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is

\n

\"state

\n\n

there are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows

\n
1
2
fullNode: [i0, i1, i2, … i15, hash]  
shortNode： [ [k0, k1, … kn], hash ] // first element is an array
\n

if the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.

\n

\"mpt\"

\n

Use the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
hex charbitspointing toodd/even2nd niddle padding
00000nodeevenno
10001nodeoddyes
20010valueevenno
30011valueoddyes
\n

this encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type

\n

In the trie module, there is a Database object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database

\n

reference

\n"},{"title":"understanding of kademlia protocol","date":"2023-01-15T03:00:30.000Z","_content":"\n# Introduction\nKademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) pairs are stored in nodes whose ID is 'close' to the key for some notion of closeness. \n\n# system description\nKad effectively treats nodes as leaves in a binary tree, with each nodes's position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.\n\n![Figure 1](/images/kademlia.subtree.png)\nfor any given node, the binary tree is divided into a series of successively lower subtrees that don't contain the node.\n The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.\n\nif there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.\n\n![Figure 2](/images/kademlia.locating.png)\n\n## node distance: XOR metric\nNode's id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.\nd(x,y) = x XOR y\n\n## node state\nfor each 0 <= i < 160, every node keeps k triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.\neach k-bucket is kept sorted by time last seen--least recently seen node at the head, most-recently seen at the tail.\nfor small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).\n\n The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8). \n\n## update of k bucket\nThere are mainly the following ways to update the K bucket:\n\n- Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.\n- Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.\n- Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.\n\nThe update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.\n\nAccording to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes\n![probability of continuous online agains onlie duration](/images/kademlia.onlineprob.png)\n\n## RPC method\nThe Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.\n\n- PING probes a node to see if it is online.\n- STORE instructs a node to store a pair for later retrieval.\n- FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator. \n- The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.\n\n## node lookup\nKad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of. \n\n## find a pair\nto find a pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.\n\n## node join\nto join the network, a node u must have a contact to an already participating node w.\n\n## routing table\n\n# references\n- [kademlia paper](https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf)\n- [go implementation](https://github.com/libp2p/go-libp2p-kad-dht)\n- [kademlia stanford](https://codethechange.stanford.edu/guides/guide_kademlia.html)\n- [zhihu](https://zhuanlan.zhihu.com/p/388994038)","source":"_posts/blockchain.kademlia.md","raw":"---\ntitle: understanding of kademlia protocol\ndate: 2023-01-15 11:00:30\ntags: [blockchain]\n---\n\n# Introduction\nKademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) pairs are stored in nodes whose ID is 'close' to the key for some notion of closeness. \n\n# system description\nKad effectively treats nodes as leaves in a binary tree, with each nodes's position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.\n\n![Figure 1](/images/kademlia.subtree.png)\nfor any given node, the binary tree is divided into a series of successively lower subtrees that don't contain the node.\n The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.\n\nif there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.\n\n![Figure 2](/images/kademlia.locating.png)\n\n## node distance: XOR metric\nNode's id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.\nd(x,y) = x XOR y\n\n## node state\nfor each 0 <= i < 160, every node keeps k triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.\neach k-bucket is kept sorted by time last seen--least recently seen node at the head, most-recently seen at the tail.\nfor small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).\n\n The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8). \n\n## update of k bucket\nThere are mainly the following ways to update the K bucket:\n\n- Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.\n- Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.\n- Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.\n\nThe update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.\n\nAccording to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes\n![probability of continuous online agains onlie duration](/images/kademlia.onlineprob.png)\n\n## RPC method\nThe Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.\n\n- PING probes a node to see if it is online.\n- STORE instructs a node to store a pair for later retrieval.\n- FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator. \n- The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.\n\n## node lookup\nKad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of. \n\n## find a pair\nto find a pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.\n\n## node join\nto join the network, a node u must have a contact to an already participating node w.\n\n## routing table\n\n# references\n- [kademlia paper](https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf)\n- [go implementation](https://github.com/libp2p/go-libp2p-kad-dht)\n- [kademlia stanford](https://codethechange.stanford.edu/guides/guide_kademlia.html)\n- [zhihu](https://zhuanlan.zhihu.com/p/388994038)","slug":"blockchain.kademlia","published":1,"updated":"2023-04-14T07:33:16.860Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpl00016hsjdgw02i4v","content":"

Introduction

Kademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) <key,value> pairs are stored in nodes whose ID is ‘close’ to the key for some notion of closeness.

\n

system description

Kad effectively treats nodes as leaves in a binary tree, with each nodes’s position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.

\n

\"Figure
for any given node, the binary tree is divided into a series of successively lower subtrees that don’t contain the node.
The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.

\n

if there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.

\n

\"Figure

\n

node distance: XOR metric

Node’s id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.
d(x,y) = x XOR y

\n

node state

for each 0 <= i < 160, every node keeps k <IP address, UDP port, node ID> triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.
each k-bucket is kept sorted by time last seen–least recently seen node at the head, most-recently seen at the tail.
for small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).

\n

The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8).

\n

update of k bucket

There are mainly the following ways to update the K bucket:

\n
    \n
  • Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.
    • \n
  • Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.
    • \n
  • Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.
    • \n
\n

The update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.

\n

According to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes
\"probability

\n

RPC method

The Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.

\n
    \n
  • PING probes a node to see if it is online.
    • \n
  • STORE instructs a node to store a <key,value> pair for later retrieval.
    • \n
  • FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator.
    • \n
  • The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.
    • \n
\n

node lookup

Kad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of.

\n

find a <key,value> pair

to find a <key,value> pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.

\n

node join

to join the network, a node u must have a contact to an already participating node w.

\n

routing table

references

\n","site":{"data":{}},"excerpt":"","more":"

Introduction

Kademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) <key,value> pairs are stored in nodes whose ID is ‘close’ to the key for some notion of closeness.

\n

system description

Kad effectively treats nodes as leaves in a binary tree, with each nodes’s position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.

\n

\"Figure
for any given node, the binary tree is divided into a series of successively lower subtrees that don’t contain the node.
The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.

\n

if there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.

\n

\"Figure

\n

node distance: XOR metric

Node’s id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.
d(x,y) = x XOR y

\n

node state

for each 0 <= i < 160, every node keeps k <IP address, UDP port, node ID> triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.
each k-bucket is kept sorted by time last seen–least recently seen node at the head, most-recently seen at the tail.
for small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).

\n

The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8).

\n

update of k bucket

There are mainly the following ways to update the K bucket:

\n
    \n
  • Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.
    • \n
  • Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.
    • \n
  • Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.
    • \n
\n

The update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.

\n

According to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes
\"probability

\n

RPC method

The Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.

\n
    \n
  • PING probes a node to see if it is online.
    • \n
  • STORE instructs a node to store a <key,value> pair for later retrieval.
    • \n
  • FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator.
    • \n
  • The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.
    • \n
\n

node lookup

Kad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of.

\n

find a <key,value> pair

to find a <key,value> pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.

\n

node join

to join the network, a node u must have a contact to an already participating node w.

\n

routing table

references

\n"},{"title":"eth_sign","date":"2022-11-28T03:47:56.000Z","_content":"\n# ethereum signing \n\n## ECDSA recap\n","source":"_posts/eth-sign.md","raw":"---\ntitle: eth_sign\ndate: 2022-11-28 11:47:56\ntags:\n---\n\n# ethereum signing \n\n## ECDSA recap\n","slug":"eth-sign","published":1,"updated":"2022-11-28T03:52:30.181Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpn00036hsjgjjl17qf","content":"

ethereum signing

ECDSA recap

","site":{"data":{}},"excerpt":"","more":"

ethereum signing

ECDSA recap

"},{"title":"geth_fine_tune","date":"2023-01-01T08:29:43.000Z","_content":"\n\nIf we're a full node on mainnet without --cache specified, bump default cache allowance\nctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))","source":"_posts/geth-fine-tune.md","raw":"---\ntitle: geth_fine_tune\ndate: 2023-01-01 16:29:43\ntags:\n---\n\n\nIf we're a full node on mainnet without --cache specified, bump default cache allowance\nctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))","slug":"geth-fine-tune","published":1,"updated":"2023-04-25T09:48:14.119Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpo00046hsjbfs3g4aw","content":"

If we’re a full node on mainnet without –cache specified, bump default cache allowance
ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))

\n","site":{"data":{}},"excerpt":"","more":"

If we’re a full node on mainnet without –cache specified, bump default cache allowance
ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))

\n"},{"title":"how to use hexo","date":"2022-11-27T03:47:56.000Z","_content":"Welcome to [Hexo](https://hexo.io/)! This is your very first post. Check [documentation](https://hexo.io/docs/) for more info. If you get any problems when using Hexo, you can find the answer in [troubleshooting](https://hexo.io/docs/troubleshooting.html) or you can ask me on [GitHub](https://github.com/hexojs/hexo/issues).\n\n## Quick Start\n\n### Create a new post\n\n``` bash\n$ hexo new \"My New Post\"\n```\n\nMore info: [Writing](https://hexo.io/docs/writing.html)\n\n### Run server\n\n``` bash\n$ hexo server\n```\n\nMore info: [Server](https://hexo.io/docs/server.html)\n\n### Generate static files\n\n``` bash\n$ hexo generate\n```\n\nMore info: [Generating](https://hexo.io/docs/generating.html)\n\n### Deploy to remote sites\n\n``` bash\n$ hexo deploy\n```\n\nMore info: [Deployment](https://hexo.io/docs/one-command-deployment.html)\n","source":"_posts/hello-world.md","raw":"---\ntitle: how to use hexo\ndate: 2022-11-27 11:47:56\n---\nWelcome to [Hexo](https://hexo.io/)! This is your very first post. Check [documentation](https://hexo.io/docs/) for more info. If you get any problems when using Hexo, you can find the answer in [troubleshooting](https://hexo.io/docs/troubleshooting.html) or you can ask me on [GitHub](https://github.com/hexojs/hexo/issues).\n\n## Quick Start\n\n### Create a new post\n\n``` bash\n$ hexo new \"My New Post\"\n```\n\nMore info: [Writing](https://hexo.io/docs/writing.html)\n\n### Run server\n\n``` bash\n$ hexo server\n```\n\nMore info: [Server](https://hexo.io/docs/server.html)\n\n### Generate static files\n\n``` bash\n$ hexo generate\n```\n\nMore info: [Generating](https://hexo.io/docs/generating.html)\n\n### Deploy to remote sites\n\n``` bash\n$ hexo deploy\n```\n\nMore info: [Deployment](https://hexo.io/docs/one-command-deployment.html)\n","slug":"hello-world","published":1,"updated":"2023-04-06T16:43:39.107Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpo00056hsj0dsfahih","content":"

Welcome to Hexo! This is your very first post. Check documentation for more info. If you get any problems when using Hexo, you can find the answer in troubleshooting or you can ask me on GitHub.

\n

Quick Start

Create a new post

1
$ hexo new "My New Post"
\n\n

More info: Writing

\n

Run server

1
$ hexo server
\n\n

More info: Server

\n

Generate static files

1
$ hexo generate
\n\n

More info: Generating

\n

Deploy to remote sites

1
$ hexo deploy
\n\n

More info: Deployment

\n","site":{"data":{}},"excerpt":"","more":"

Welcome to Hexo! This is your very first post. Check documentation for more info. If you get any problems when using Hexo, you can find the answer in troubleshooting or you can ask me on GitHub.

\n

Quick Start

Create a new post

1
$ hexo new "My New Post"
\n\n

More info: Writing

\n

Run server

1
$ hexo server
\n\n

More info: Server

\n

Generate static files

1
$ hexo generate
\n\n

More info: Generating

\n

Deploy to remote sites

1
$ hexo deploy
\n\n

More info: Deployment

\n"},{"title":"go similar concepts comparison","date":"2023-03-05T02:44:50.000Z","_content":"\n## struct{} & struct{}{}\n`struct` is a keyword in Go. It is used to define struct types, which is a sequence of named elements.\n\nFor example:\n```go\ntype Person struct {\n Name string\n Age int\n}\n```\nThe `struct{}` is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type `struct{}`.\n\n`struct{}{}` on the other hand is a [composite literal](https://go.dev/ref/spec#Composite_literals), it constructs a value of type `struct{}`. A composite literal constructs values for types such as `structs`, `arrays`, `maps` and `slices`. Its syntax is the type followed by the elements in braces. Since the \"empty\" struct (struct{}) has no fields, the elements list is also empty:\n\n struct{} {}\n| ^ | ^\n type empty element list\nAs an example let's create a \"set\" in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.\n\nA map with string elements:\n```go\nvar set map[string]struct{}\n// Initialize the set\nset = make(map[string]struct{})\n\n// Add some values to the set:\nset[\"red\"] = struct{}{}\nset[\"blue\"] = struct{}{}\n\n// Check if a value is in the map:\n_, ok := set[\"red\"]\nfmt.Println(\"Is red in the map?\", ok)\n_, ok = set[\"green\"]\nfmt.Println(\"Is green in the map?\", ok)\n```","source":"_posts/golang/go-similar-concepts-comparison.md","raw":"---\ntitle: go similar concepts comparison\ndate: 2023-03-05 10:44:50\ntags: [golang]\n---\n\n## struct{} & struct{}{}\n`struct` is a keyword in Go. It is used to define struct types, which is a sequence of named elements.\n\nFor example:\n```go\ntype Person struct {\n Name string\n Age int\n}\n```\nThe `struct{}` is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type `struct{}`.\n\n`struct{}{}` on the other hand is a [composite literal](https://go.dev/ref/spec#Composite_literals), it constructs a value of type `struct{}`. A composite literal constructs values for types such as `structs`, `arrays`, `maps` and `slices`. Its syntax is the type followed by the elements in braces. Since the \"empty\" struct (struct{}) has no fields, the elements list is also empty:\n\n struct{} {}\n| ^ | ^\n type empty element list\nAs an example let's create a \"set\" in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.\n\nA map with string elements:\n```go\nvar set map[string]struct{}\n// Initialize the set\nset = make(map[string]struct{})\n\n// Add some values to the set:\nset[\"red\"] = struct{}{}\nset[\"blue\"] = struct{}{}\n\n// Check if a value is in the map:\n_, ok := set[\"red\"]\nfmt.Println(\"Is red in the map?\", ok)\n_, ok = set[\"green\"]\nfmt.Println(\"Is green in the map?\", ok)\n```","slug":"golang/go-similar-concepts-comparison","published":1,"updated":"2023-06-18T07:07:22.116Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpp00076hsjh7rifext","content":"

struct{} & struct{}{}

struct is a keyword in Go. It is used to define struct types, which is a sequence of named elements.

\n

For example:

\n
1
2
3
4
type Person struct {
    Name string
    Age  int
}
\n

The struct{} is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type struct{}.

\n

struct{}{} on the other hand is a composite literal, it constructs a value of type struct{}. A composite literal constructs values for types such as structs, arrays, maps and slices. Its syntax is the type followed by the elements in braces. Since the “empty” struct (struct{}) has no fields, the elements list is also empty:

\n

struct{} {}
| ^ | ^
type empty element list
As an example let’s create a “set” in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.

\n

A map with string elements:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
var set map[string]struct{}
// Initialize the set
set = make(map[string]struct{})

// Add some values to the set:
set["red"] = struct{}{}
set["blue"] = struct{}{}

// Check if a value is in the map:
_, ok := set["red"]
fmt.Println("Is red in the map?", ok)
_, ok = set["green"]
fmt.Println("Is green in the map?", ok)
","site":{"data":{}},"excerpt":"","more":"

struct{} & struct{}{}

struct is a keyword in Go. It is used to define struct types, which is a sequence of named elements.

\n

For example:

\n
1
2
3
4
type Person struct {
    Name string
    Age  int
}
\n

The struct{} is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type struct{}.

\n

struct{}{} on the other hand is a composite literal, it constructs a value of type struct{}. A composite literal constructs values for types such as structs, arrays, maps and slices. Its syntax is the type followed by the elements in braces. Since the “empty” struct (struct{}) has no fields, the elements list is also empty:

\n

struct{} {}
| ^ | ^
type empty element list
As an example let’s create a “set” in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.

\n

A map with string elements:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
var set map[string]struct{}
// Initialize the set
set = make(map[string]struct{})

// Add some values to the set:
set["red"] = struct{}{}
set["blue"] = struct{}{}

// Check if a value is in the map:
_, ok := set["red"]
fmt.Println("Is red in the map?", ok)
_, ok = set["green"]
fmt.Println("Is green in the map?", ok)
"},{"title":"go reflect","date":"2023-03-02T02:44:50.000Z","_content":"\n## introduction\nReflection is the ability of a program to examine its own structure, particularly through types. \n\n## type and interfaces\nGo is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare\n```go\ntype MyInt int\n\nvar i int\nvar j MyInt\n```\n**The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.**\n\nOne important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:\n```go\ninterface{}\n```\nor its equivalent alias,\n```go\nany\n```\nIt represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.\na variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.\n\n## the representation of an interface\nA variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.\nFor instance, after\n```golang\nvar r io.Reader\ntty, err := os.OpenFile(\"/dev/tty\", os.O_RDWR, 0)\nif err != nil {\n return nil, err\n}\nr = tty\n```\nr contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:\n```go\nvar w io.Writer\nw = r.(io.Writer)\n```\nThe expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r. \nOne important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.\n\n## the first law of reflection\n1. Reflection goes from interface value to reflection object\nAt the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. `reflect.TypeOf` and `reflect.ValueOf`, retrieve `reflect.Type` and `reflect.Value` pieces out of an interface value.\n```go\nvar x float64 = 3.4\nfmt.Println(\"type:\", reflect.TypeOf(x))\n```\n```\ntype: float64\n```\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type())\nfmt.Println(\"kind is float64:\", v.Kind() == reflect.Float64)\nfmt.Println(\"value:\", v.Float())\n```\n```\ntype: float64\nkind is float64: true\nvalue: 3.4\n```\nThere are also methods like SetInt and SetFloat. **to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value**: int64 for all the signed integers, for instance. \n```go\nvar x uint8 = 'x'\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type()) // uint8.\nfmt.Println(\"kind is uint8: \", v.Kind() == reflect.Uint8) // true.\nx = uint8(v.Uint()) // v.Uint returns a uint64.\n```\n\n2. Reflection goes from reflection object to interface value.\nGiven a reflect.Value we can recover an interface value using the Interface method;\n```go\n// Interface returns v's current value as an interface{}.\n// It is equivalent to:\n//\n//\tvar i interface{} = (v's underlying value)\nfunc (v Value) Interface() interface{}\n```\n```go\ny := v.Interface().(float64) // y will have type float64.\nfmt.Println(y)\n```\n\n3. To modify a reflection object, the value must be settable.\nThe CanSet method of Value reports the settability of a Value; in our case,\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: false\n```\nwe pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement `v.SetFloat(7.1)` were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.\nLet’s do that. \n```go\nvar x float64 = 3.4\np := reflect.ValueOf(&x) // Note: take the address of x.\nfmt.Println(\"type of p:\", p.Type())\nfmt.Println(\"settability of p:\", p.CanSet())\n```\nThe reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:\n```go\nv := p.Elem()\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: true\n```\n\n## structs\n\n```go\ntype T struct {\n A int\n B string\n}\nt := T{23, \"skidoo\"}\ns := reflect.ValueOf(&t).Elem()\ntypeOfT := s.Type()\nfor i := 0; i < s.NumField(); i++ {\n f := s.Field(i)\n fmt.Printf(\"%d: %s %s = %v\\n\", i,\n typeOfT.Field(i).Name, f.Type(), f.Interface())\n}\n```\n```\n0: A int = 23\n1: B string = skidoo\n```\nThere’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.\nBecause s contains a settable reflection object, we can modify the fields of the structure.\n```\ns.Field(0).SetInt(77)\ns.Field(1).SetString(\"Sunset Strip\")\nfmt.Println(\"t is now\", t)\n```\nIf we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.\n\n## references\n- [official blog](https://go.dev/blog/laws-of-reflection)\n- [go data structure: interface](https://research.swtch.com/interfaces)","source":"_posts/golang/go-reflect.md","raw":"---\ntitle: go reflect\ndate: 2023-03-02 10:44:50\ntags: [golang]\n---\n\n## introduction\nReflection is the ability of a program to examine its own structure, particularly through types. \n\n## type and interfaces\nGo is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare\n```go\ntype MyInt int\n\nvar i int\nvar j MyInt\n```\n**The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.**\n\nOne important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:\n```go\ninterface{}\n```\nor its equivalent alias,\n```go\nany\n```\nIt represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.\na variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.\n\n## the representation of an interface\nA variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.\nFor instance, after\n```golang\nvar r io.Reader\ntty, err := os.OpenFile(\"/dev/tty\", os.O_RDWR, 0)\nif err != nil {\n return nil, err\n}\nr = tty\n```\nr contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:\n```go\nvar w io.Writer\nw = r.(io.Writer)\n```\nThe expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r. \nOne important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.\n\n## the first law of reflection\n1. Reflection goes from interface value to reflection object\nAt the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. `reflect.TypeOf` and `reflect.ValueOf`, retrieve `reflect.Type` and `reflect.Value` pieces out of an interface value.\n```go\nvar x float64 = 3.4\nfmt.Println(\"type:\", reflect.TypeOf(x))\n```\n```\ntype: float64\n```\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type())\nfmt.Println(\"kind is float64:\", v.Kind() == reflect.Float64)\nfmt.Println(\"value:\", v.Float())\n```\n```\ntype: float64\nkind is float64: true\nvalue: 3.4\n```\nThere are also methods like SetInt and SetFloat. **to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value**: int64 for all the signed integers, for instance. \n```go\nvar x uint8 = 'x'\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type()) // uint8.\nfmt.Println(\"kind is uint8: \", v.Kind() == reflect.Uint8) // true.\nx = uint8(v.Uint()) // v.Uint returns a uint64.\n```\n\n2. Reflection goes from reflection object to interface value.\nGiven a reflect.Value we can recover an interface value using the Interface method;\n```go\n// Interface returns v's current value as an interface{}.\n// It is equivalent to:\n//\n//\tvar i interface{} = (v's underlying value)\nfunc (v Value) Interface() interface{}\n```\n```go\ny := v.Interface().(float64) // y will have type float64.\nfmt.Println(y)\n```\n\n3. To modify a reflection object, the value must be settable.\nThe CanSet method of Value reports the settability of a Value; in our case,\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: false\n```\nwe pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement `v.SetFloat(7.1)` were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.\nLet’s do that. \n```go\nvar x float64 = 3.4\np := reflect.ValueOf(&x) // Note: take the address of x.\nfmt.Println(\"type of p:\", p.Type())\nfmt.Println(\"settability of p:\", p.CanSet())\n```\nThe reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:\n```go\nv := p.Elem()\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: true\n```\n\n## structs\n\n```go\ntype T struct {\n A int\n B string\n}\nt := T{23, \"skidoo\"}\ns := reflect.ValueOf(&t).Elem()\ntypeOfT := s.Type()\nfor i := 0; i < s.NumField(); i++ {\n f := s.Field(i)\n fmt.Printf(\"%d: %s %s = %v\\n\", i,\n typeOfT.Field(i).Name, f.Type(), f.Interface())\n}\n```\n```\n0: A int = 23\n1: B string = skidoo\n```\nThere’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.\nBecause s contains a settable reflection object, we can modify the fields of the structure.\n```\ns.Field(0).SetInt(77)\ns.Field(1).SetString(\"Sunset Strip\")\nfmt.Println(\"t is now\", t)\n```\nIf we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.\n\n## references\n- [official blog](https://go.dev/blog/laws-of-reflection)\n- [go data structure: interface](https://research.swtch.com/interfaces)","slug":"golang/go-reflect","published":1,"updated":"2023-06-18T06:42:44.968Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpp00086hsjfqu01qqy","content":"

introduction

Reflection is the ability of a program to examine its own structure, particularly through types.

\n

type and interfaces

Go is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare

\n
1
2
3
4
type MyInt int

var i int
var j MyInt
\n

The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.

\n

One important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:

\n
1
interface{}
\n

or its equivalent alias,

\n
1
any
\n

It represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.
a variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.

\n

the representation of an interface

A variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.
For instance, after

\n
1
2
3
4
5
6
var r io.Reader
tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
if err != nil {
    return nil, err
}
r = tty
\n

r contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:

\n
1
2
var w io.Writer
w = r.(io.Writer)
\n

The expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r.
One important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.

\n

the first law of reflection

    \n

  1. Reflection goes from interface value to reflection object
    At the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. reflect.TypeOf and reflect.ValueOf, retrieve reflect.Type and reflect.Value pieces out of an interface value.

    \n
    1
    2
    var x float64 = 3.4
    fmt.Println("type:", reflect.TypeOf(x))
    \n
    1
    type: float64
    \n
    1
    2
    3
    4
    5
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())
    fmt.Println("kind is float64:", v.Kind() == reflect.Float64)
    fmt.Println("value:", v.Float())
    \n
    1
    2
    3
    type: float64
    kind is float64: true
    value: 3.4
    \n

    There are also methods like SetInt and SetFloat. to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value: int64 for all the signed integers, for instance.

    \n
    1
    2
    3
    4
    5
    var x uint8 = 'x'
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())                            // uint8.
    fmt.Println("kind is uint8: ", v.Kind() == reflect.Uint8) // true.
    x = uint8(v.Uint())                                       // v.Uint returns a uint64.
    \n
    2. \n

  2. Reflection goes from reflection object to interface value.
    Given a reflect.Value we can recover an interface value using the Interface method;

    \n
    1
    2
    3
    4
    5
    // Interface returns v's current value as an interface{}.
    // It is equivalent to:
    //
    //\tvar i interface{} = (v's underlying value)
    func (v Value) Interface() interface{}
    \n
    1
    2
    y := v.Interface().(float64) // y will have type float64.
    fmt.Println(y)
    \n
    3. \n

  3. To modify a reflection object, the value must be settable.
    The CanSet method of Value reports the settability of a Value; in our case,

    \n
    1
    2
    3
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: false
    \n

    we pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement v.SetFloat(7.1) were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.
    Let’s do that.

    \n
    1
    2
    3
    4
    var x float64 = 3.4
    p := reflect.ValueOf(&x) // Note: take the address of x.
    fmt.Println("type of p:", p.Type())
    fmt.Println("settability of p:", p.CanSet())
    \n

    The reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:

    \n
    1
    2
    v := p.Elem()
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: true
    4. \n
\n

structs

1
2
3
4
5
6
7
8
9
10
11
12
type T struct {
    A int
    B string
}
t := T{23, "skidoo"}
s := reflect.ValueOf(&t).Elem()
typeOfT := s.Type()
for i := 0; i < s.NumField(); i++ {
    f := s.Field(i)
    fmt.Printf("%d: %s %s = %v\\n", i,
        typeOfT.Field(i).Name, f.Type(), f.Interface())
}
\n
1
2
0: A int = 23
1: B string = skidoo
\n

There’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.
Because s contains a settable reflection object, we can modify the fields of the structure.

\n
1
2
3
s.Field(0).SetInt(77)
s.Field(1).SetString("Sunset Strip")
fmt.Println("t is now", t)
\n

If we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

Reflection is the ability of a program to examine its own structure, particularly through types.

\n

type and interfaces

Go is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare

\n
1
2
3
4
type MyInt int

var i int
var j MyInt
\n

The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.

\n

One important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:

\n
1
interface{}
\n

or its equivalent alias,

\n
1
any
\n

It represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.
a variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.

\n

the representation of an interface

A variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.
For instance, after

\n
1
2
3
4
5
6
var r io.Reader
tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
if err != nil {
    return nil, err
}
r = tty
\n

r contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:

\n
1
2
var w io.Writer
w = r.(io.Writer)
\n

The expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r.
One important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.

\n

the first law of reflection

    \n

  1. Reflection goes from interface value to reflection object
    At the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. reflect.TypeOf and reflect.ValueOf, retrieve reflect.Type and reflect.Value pieces out of an interface value.

    \n
    1
    2
    var x float64 = 3.4
    fmt.Println("type:", reflect.TypeOf(x))
    \n
    1
    type: float64
    \n
    1
    2
    3
    4
    5
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())
    fmt.Println("kind is float64:", v.Kind() == reflect.Float64)
    fmt.Println("value:", v.Float())
    \n
    1
    2
    3
    type: float64
    kind is float64: true
    value: 3.4
    \n

    There are also methods like SetInt and SetFloat. to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value: int64 for all the signed integers, for instance.

    \n
    1
    2
    3
    4
    5
    var x uint8 = 'x'
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())                            // uint8.
    fmt.Println("kind is uint8: ", v.Kind() == reflect.Uint8) // true.
    x = uint8(v.Uint())                                       // v.Uint returns a uint64.
    \n
    2. \n

  2. Reflection goes from reflection object to interface value.
    Given a reflect.Value we can recover an interface value using the Interface method;

    \n
    1
    2
    3
    4
    5
    // Interface returns v's current value as an interface{}.
    // It is equivalent to:
    //
    //\tvar i interface{} = (v's underlying value)
    func (v Value) Interface() interface{}
    \n
    1
    2
    y := v.Interface().(float64) // y will have type float64.
    fmt.Println(y)
    \n
    3. \n

  3. To modify a reflection object, the value must be settable.
    The CanSet method of Value reports the settability of a Value; in our case,

    \n
    1
    2
    3
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: false
    \n

    we pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement v.SetFloat(7.1) were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.
    Let’s do that.

    \n
    1
    2
    3
    4
    var x float64 = 3.4
    p := reflect.ValueOf(&x) // Note: take the address of x.
    fmt.Println("type of p:", p.Type())
    fmt.Println("settability of p:", p.CanSet())
    \n

    The reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:

    \n
    1
    2
    v := p.Elem()
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: true
    4. \n
\n

structs

1
2
3
4
5
6
7
8
9
10
11
12
type T struct {
    A int
    B string
}
t := T{23, "skidoo"}
s := reflect.ValueOf(&t).Elem()
typeOfT := s.Type()
for i := 0; i < s.NumField(); i++ {
    f := s.Field(i)
    fmt.Printf("%d: %s %s = %v\\n", i,
        typeOfT.Field(i).Name, f.Type(), f.Interface())
}
\n
1
2
0: A int = 23
1: B string = skidoo
\n

There’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.
Because s contains a settable reflection object, we can modify the fields of the structure.

\n
1
2
3
s.Field(0).SetInt(77)
s.Field(1).SetString("Sunset Strip")
fmt.Println("t is now", t)
\n

If we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.

\n

references

\n"},{"title":"rust resource","date":"2022-09-27T14:04:38.000Z","_content":"\n## learning resources\n- [the book](https://doc.rust-lang.org/book/)\n- [cargo book](https://doc.rust-lang.org/cargo/index.html)\n- [the rust performance book](https://nnethercote.github.io/perf-book/title-page.html)\n- [rust design patterns](https://rust-unofficial.github.io/patterns/intro.html)\n- [the rust RFC book](https://rust-lang.github.io/rfcs/)\n- [the rustdoc book](https://doc.rust-lang.org/rustdoc/what-is-rustdoc.html)\n- [rustnomicon](https://doc.rust-lang.org/nomicon/intro.html)\n- [the rust reference](https://doc.rust-lang.org/reference/introduction.html)\n- [rust by example](https://doc.rust-lang.org/rust-by-example/index.html)\n- [async programming in rust](https://rust-lang.github.io/async-book/01_getting_started/01_chapter.html)\n- [rust compiler development guide](https://rustc-dev-guide.rust-lang.org/about-this-guide.html)\n- [the little book of rust macros](https://veykril.github.io/tlborm/)\n\n## video resources\n- [youtube channel](https://www.youtube.com/@jonhoo)\n\n## books\n- [atomics and locks](https://marabos.nl/atomics/)\n- [data structure & algorithm](https://github.com/QMHTMY/RustBook/blob/main/books/rust-book-zh-cn-shieber.pdf)","source":"_posts/rust/rust-01-resource.md","raw":"---\ntitle: rust resource\ndate: 2022-09-27 22:04:38\ntags: [rust]\n---\n\n## learning resources\n- [the book](https://doc.rust-lang.org/book/)\n- [cargo book](https://doc.rust-lang.org/cargo/index.html)\n- [the rust performance book](https://nnethercote.github.io/perf-book/title-page.html)\n- [rust design patterns](https://rust-unofficial.github.io/patterns/intro.html)\n- [the rust RFC book](https://rust-lang.github.io/rfcs/)\n- [the rustdoc book](https://doc.rust-lang.org/rustdoc/what-is-rustdoc.html)\n- [rustnomicon](https://doc.rust-lang.org/nomicon/intro.html)\n- [the rust reference](https://doc.rust-lang.org/reference/introduction.html)\n- [rust by example](https://doc.rust-lang.org/rust-by-example/index.html)\n- [async programming in rust](https://rust-lang.github.io/async-book/01_getting_started/01_chapter.html)\n- [rust compiler development guide](https://rustc-dev-guide.rust-lang.org/about-this-guide.html)\n- [the little book of rust macros](https://veykril.github.io/tlborm/)\n\n## video resources\n- [youtube channel](https://www.youtube.com/@jonhoo)\n\n## books\n- [atomics and locks](https://marabos.nl/atomics/)\n- [data structure & algorithm](https://github.com/QMHTMY/RustBook/blob/main/books/rust-book-zh-cn-shieber.pdf)","slug":"rust/rust-01-resource","published":1,"updated":"2023-06-29T04:06:05.747Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpq000a6hsj4hyvg73a","content":"

learning resources

\n

video resources

\n

books

\n","site":{"data":{}},"excerpt":"","more":"

learning resources

\n

video resources

\n

books

\n"},{"title":"rust ownership","date":"2022-10-11T09:09:28.000Z","_content":"\n## ownership rule\n- Each value in Rust has an owner.\n- There can only be one owner at a time.\n- When the owner goes out of scope, the value will be dropped.\n\n## variable scope\n```rust\nfn main() {\n { // s is not valid here, it’s not yet declared\n let s = \"hello\"; // s is valid from this point forward\n // do stuff with s\n } // this scope is now over, and s is no longer valid\n}\n```\n\n## Move\n### stack-only data: Copy trait\nsuch as primitive type\n```rust\nfn main() {\n let x = 5;\n let y = x;\n}\n```\nbind the value 5 to x; then make a copy of the value in x and bind it to y.\n\n### for heap variable\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1;\n}\n```\n![move-string](/images/rust/ownership/move-string.png)\nptr, len, capacity is stored in stack, while string value is stored in heap \\\nWhen we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap\n![move-string-2](/images/rust/ownership/move-string-2.png)\nsimilar to shallow copy\n\n## Clone\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1.clone();\n\n println!(\"s1 = {}, s2 = {}\", s1, s2);\n}\n```\nIf we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.\n\n### ownership and function\n- Passing a value to a function will result in a move or copy of ownership\n- difference between \"stack\" and \"heap\" variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable\n```rust\nfn main() {\n let s = String::from(\"hello\"); // s comes into scope\n takes_ownership(s); // s's value moves into the function...\n // ... and so is no longer valid here\n\n let x = 5; // x comes into scope\n\n makes_copy(x); // x would move into the function,\n // but i32 is Copy, so it's okay to still\n // use x afterward\n\n} // Here, x goes out of scope, then s. But because s's value was moved, nothing\n // special happens.\n\nfn takes_ownership(some_string: String) { // some_string comes into scope\n println!(\"{}\", some_string);\n} // Here, some_string goes out of scope and `drop` is called. The backing\n // memory is freed.\n\nfn makes_copy(some_integer: i32) { // some_integer comes into scope\n println!(\"{}\", some_integer);\n} // Here, some_integer goes out of scope. Nothing special happens.\n\n```\n\n### return values and scope\n```rust\nfn main() {\n let s1 = gives_ownership(); // gives_ownership moves its return\n // value into s1\n\n let s2 = String::from(\"hello\"); // s2 comes into scope\n\n let s3 = takes_and_gives_back(s2); // s2 is moved into\n // takes_and_gives_back, which also\n // moves its return value into s3\n} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing\n // happens. s1 goes out of scope and is dropped.\n\nfn gives_ownership() -> String { // gives_ownership will move its\n // return value into the function\n // that calls it\n\n let some_string = String::from(\"yours\"); // some_string comes into scope\n\n some_string // some_string is returned and\n // moves out to the calling\n // function\n}\n\n// This function takes a String and returns one\nfn takes_and_gives_back(a_string: String) -> String { // a_string comes into\n // scope\n\n a_string // a_string is returned and moves out to the calling function\n}\n```\n> What if we want to let a function use a value but not take ownership? \n> that's reference\n \n\n## reference & borrow\n- & means reference (borrow but not own)， default immutable\n- &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)\n- Multiple mutable references can be created non-simultaneously by creating a new scope\n- **Cannot have mutable and immutable references at the same time**\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n {\n let s1 = &mut s;\n } // r1 goes out of scope here, so we can make a new reference with no problems.\n let s2 = &mut s;\n}\n\nfn main() {\n let mut s = String::from(\"hello\");\n\n let r1 = &s; // no problem\n let r2 = &s; // no problem\n println!(\"{} and {}\", r1, r2);\n // variables r1 and r2 will not be used after this point\n\n let r3 = &mut s; // no problem\n println!(\"{}\", r3);\n\n println!{\"{}\",r1} // got problem with above mutable borrow\n}\n```\n\n### reference as function arguments\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n\n let len = calculate_length(&s1);\n\n println!(\"The length of '{}' is {}.\", s1, len);\n}\n\nfn calculate_length(s: &String) -> usize { // s is a reference to a String\n s.len() \n} // Here, s goes out of scope. But because it does not have ownership of what\n // it refers to, nothing happens.\n```\nwe pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used. \\\n When functions have references as parameters instead of the actual values, we won't need to return the values in order to give back ownership, because we never had ownership. \\\n We call the action of creating a reference borrowing. \n\n### mutable references\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n\n change(&mut s);\n}\n\nfn change(some_string: &mut String) {\n some_string.push_str(\", world\");\n}\n```\n\n### dangling references\n- A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else\n- rust，The compiler can guarantee that there will never be dangling references\n```rust\nfn main() {\n let r = dangle();\n}\nfn dangle() -> &string { // dangle returns a reference to a String\n let s = String::from(\"hello\"); // s is a new String\n &s // we return a reference to the String, s\n} // Here, s goes out of scope, and is dropped. Its memory goes away.\n// Danger\n```\nThe solution here is to return the String directly:\n```rust\nfn main() {\n let string = no_dangle();\n}\n\nfn no_dangle() -> String {\n let s = String::from(\"hello\");\n s\n}\n```\nThis works without any problems. Ownership is moved out, and nothing is deallocated.\n\n\n## slice\n- Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.\n\n### string slice\n```rust\nfn main() {\n let mut s = String::from(\"Hello world\");\n\n let hello = &s[0..5]; \n let world = &s[6..11];\n}\n```\nRather than a reference to the entire String, hello is a reference to a portion of the String, \\\nWith Rust's `..` range syntax, if you want to start at index zero, you can drop the value before the two periods \\\nBy the same token, if your slice includes the last byte of the String, you can drop the trailing number. \n> Note: String slice range indices must occur at **valid UTF-8 character boundaries**. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.\n\n```rust\nfn first_word(s :&String) -> &str {\n let bytes = s.as_bytes();\n for(i, &item) in bytes.iter().enumerate() {\n if item == b' ' {\n return &s[..i];\n }\n }\n &s[..]\n}\n```\n\n### String Literals Are Slices\n```rust\nfn main() {\nlet s = \"Hello, world!\";\n}\n```\nThe type of s here is &str: it's a slice pointing to that specific point of the binary. \n\n### String Slices as Parameters\n- Pass &str as a parameter, you can receive parameters of type &String and &str at the same time\n```rust\nfn first_word(s: &String) -> &str\n```\nequivalent to\n```rust\nfn first_word(s: &str) -> &str\n```\n\n### other slices\narray slice\n```rust\nfn main() {\n let a = [1, 2, 3, 4, 5];\n let slice = &a[1..3];\n assert_eq!(slice, &[2, 3]);\n}\n```","source":"_posts/rust/rust-03-ownership.md","raw":"---\ntitle: rust ownership\ndate: 2022-10-11 17:09:28\ntags: [rust]\n---\n\n## ownership rule\n- Each value in Rust has an owner.\n- There can only be one owner at a time.\n- When the owner goes out of scope, the value will be dropped.\n\n## variable scope\n```rust\nfn main() {\n { // s is not valid here, it’s not yet declared\n let s = \"hello\"; // s is valid from this point forward\n // do stuff with s\n } // this scope is now over, and s is no longer valid\n}\n```\n\n## Move\n### stack-only data: Copy trait\nsuch as primitive type\n```rust\nfn main() {\n let x = 5;\n let y = x;\n}\n```\nbind the value 5 to x; then make a copy of the value in x and bind it to y.\n\n### for heap variable\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1;\n}\n```\n![move-string](/images/rust/ownership/move-string.png)\nptr, len, capacity is stored in stack, while string value is stored in heap \\\nWhen we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap\n![move-string-2](/images/rust/ownership/move-string-2.png)\nsimilar to shallow copy\n\n## Clone\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1.clone();\n\n println!(\"s1 = {}, s2 = {}\", s1, s2);\n}\n```\nIf we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.\n\n### ownership and function\n- Passing a value to a function will result in a move or copy of ownership\n- difference between \"stack\" and \"heap\" variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable\n```rust\nfn main() {\n let s = String::from(\"hello\"); // s comes into scope\n takes_ownership(s); // s's value moves into the function...\n // ... and so is no longer valid here\n\n let x = 5; // x comes into scope\n\n makes_copy(x); // x would move into the function,\n // but i32 is Copy, so it's okay to still\n // use x afterward\n\n} // Here, x goes out of scope, then s. But because s's value was moved, nothing\n // special happens.\n\nfn takes_ownership(some_string: String) { // some_string comes into scope\n println!(\"{}\", some_string);\n} // Here, some_string goes out of scope and `drop` is called. The backing\n // memory is freed.\n\nfn makes_copy(some_integer: i32) { // some_integer comes into scope\n println!(\"{}\", some_integer);\n} // Here, some_integer goes out of scope. Nothing special happens.\n\n```\n\n### return values and scope\n```rust\nfn main() {\n let s1 = gives_ownership(); // gives_ownership moves its return\n // value into s1\n\n let s2 = String::from(\"hello\"); // s2 comes into scope\n\n let s3 = takes_and_gives_back(s2); // s2 is moved into\n // takes_and_gives_back, which also\n // moves its return value into s3\n} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing\n // happens. s1 goes out of scope and is dropped.\n\nfn gives_ownership() -> String { // gives_ownership will move its\n // return value into the function\n // that calls it\n\n let some_string = String::from(\"yours\"); // some_string comes into scope\n\n some_string // some_string is returned and\n // moves out to the calling\n // function\n}\n\n// This function takes a String and returns one\nfn takes_and_gives_back(a_string: String) -> String { // a_string comes into\n // scope\n\n a_string // a_string is returned and moves out to the calling function\n}\n```\n> What if we want to let a function use a value but not take ownership? \n> that's reference\n \n\n## reference & borrow\n- & means reference (borrow but not own)， default immutable\n- &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)\n- Multiple mutable references can be created non-simultaneously by creating a new scope\n- **Cannot have mutable and immutable references at the same time**\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n {\n let s1 = &mut s;\n } // r1 goes out of scope here, so we can make a new reference with no problems.\n let s2 = &mut s;\n}\n\nfn main() {\n let mut s = String::from(\"hello\");\n\n let r1 = &s; // no problem\n let r2 = &s; // no problem\n println!(\"{} and {}\", r1, r2);\n // variables r1 and r2 will not be used after this point\n\n let r3 = &mut s; // no problem\n println!(\"{}\", r3);\n\n println!{\"{}\",r1} // got problem with above mutable borrow\n}\n```\n\n### reference as function arguments\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n\n let len = calculate_length(&s1);\n\n println!(\"The length of '{}' is {}.\", s1, len);\n}\n\nfn calculate_length(s: &String) -> usize { // s is a reference to a String\n s.len() \n} // Here, s goes out of scope. But because it does not have ownership of what\n // it refers to, nothing happens.\n```\nwe pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used. \\\n When functions have references as parameters instead of the actual values, we won't need to return the values in order to give back ownership, because we never had ownership. \\\n We call the action of creating a reference borrowing. \n\n### mutable references\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n\n change(&mut s);\n}\n\nfn change(some_string: &mut String) {\n some_string.push_str(\", world\");\n}\n```\n\n### dangling references\n- A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else\n- rust，The compiler can guarantee that there will never be dangling references\n```rust\nfn main() {\n let r = dangle();\n}\nfn dangle() -> &string { // dangle returns a reference to a String\n let s = String::from(\"hello\"); // s is a new String\n &s // we return a reference to the String, s\n} // Here, s goes out of scope, and is dropped. Its memory goes away.\n// Danger\n```\nThe solution here is to return the String directly:\n```rust\nfn main() {\n let string = no_dangle();\n}\n\nfn no_dangle() -> String {\n let s = String::from(\"hello\");\n s\n}\n```\nThis works without any problems. Ownership is moved out, and nothing is deallocated.\n\n\n## slice\n- Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.\n\n### string slice\n```rust\nfn main() {\n let mut s = String::from(\"Hello world\");\n\n let hello = &s[0..5]; \n let world = &s[6..11];\n}\n```\nRather than a reference to the entire String, hello is a reference to a portion of the String, \\\nWith Rust's `..` range syntax, if you want to start at index zero, you can drop the value before the two periods \\\nBy the same token, if your slice includes the last byte of the String, you can drop the trailing number. \n> Note: String slice range indices must occur at **valid UTF-8 character boundaries**. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.\n\n```rust\nfn first_word(s :&String) -> &str {\n let bytes = s.as_bytes();\n for(i, &item) in bytes.iter().enumerate() {\n if item == b' ' {\n return &s[..i];\n }\n }\n &s[..]\n}\n```\n\n### String Literals Are Slices\n```rust\nfn main() {\nlet s = \"Hello, world!\";\n}\n```\nThe type of s here is &str: it's a slice pointing to that specific point of the binary. \n\n### String Slices as Parameters\n- Pass &str as a parameter, you can receive parameters of type &String and &str at the same time\n```rust\nfn first_word(s: &String) -> &str\n```\nequivalent to\n```rust\nfn first_word(s: &str) -> &str\n```\n\n### other slices\narray slice\n```rust\nfn main() {\n let a = [1, 2, 3, 4, 5];\n let slice = &a[1..3];\n assert_eq!(slice, &[2, 3]);\n}\n```","slug":"rust/rust-03-ownership","published":1,"updated":"2023-05-01T13:17:32.602Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpq000c6hsj11cxhldu","content":"

ownership rule

    \n
  • Each value in Rust has an owner.
    • \n
  • There can only be one owner at a time.
    • \n
  • When the owner goes out of scope, the value will be dropped.
    • \n
\n

variable scope

1
2
3
4
5
6
fn main() {
    {                      // s is not valid here, it’s not yet declared
        let s = "hello";   // s is valid from this point forward
        // do stuff with s
    }                      // this scope is now over, and s is no longer valid
}
\n\n

Move

stack-only data: Copy trait

such as primitive type

\n
1
2
3
4
fn main() {
    let x = 5;
    let y = x;
}
\n

bind the value 5 to x; then make a copy of the value in x and bind it to y.

\n

for heap variable

1
2
3
4
fn main() {
    let s1 = String::from("hello");
    let s2 = s1;
}
\n

\"move-string\"
ptr, len, capacity is stored in stack, while string value is stored in heap
When we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap
\"move-string-2\"
similar to shallow copy

\n

Clone

1
2
3
4
5
6
fn main() {
    let s1 = String::from("hello");
    let s2 = s1.clone();

    println!("s1 = {}, s2 = {}", s1, s2);
}
\n

If we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.

\n

ownership and function

    \n
  • Passing a value to a function will result in a move or copy of ownership
    • \n
  • difference between “stack” and “heap” variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    fn main() {
        let s = String::from("hello");  // s comes into scope
        takes_ownership(s);             // s's value moves into the function...
                                        // ... and so is no longer valid here

        let x = 5;                      // x comes into scope

        makes_copy(x);                  // x would move into the function,
                                        // but i32 is Copy, so it's okay to still
                                        // use x afterward

    } // Here, x goes out of scope, then s. But because s's value was moved, nothing
      // special happens.

    fn takes_ownership(some_string: String) { // some_string comes into scope
        println!("{}", some_string);
    } // Here, some_string goes out of scope and `drop` is called. The backing
      // memory is freed.

    fn makes_copy(some_integer: i32) { // some_integer comes into scope
        println!("{}", some_integer);
    } // Here, some_integer goes out of scope. Nothing special happens.

    • \n
\n

return values and scope

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
fn main() {
    let s1 = gives_ownership();         // gives_ownership moves its return
                                        // value into s1

    let s2 = String::from("hello");     // s2 comes into scope

    let s3 = takes_and_gives_back(s2);  // s2 is moved into
                                        // takes_and_gives_back, which also
                                        // moves its return value into s3
} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing
  // happens. s1 goes out of scope and is dropped.

fn gives_ownership() -> String {             // gives_ownership will move its
                                             // return value into the function
                                             // that calls it

    let some_string = String::from("yours"); // some_string comes into scope

    some_string                              // some_string is returned and
                                             // moves out to the calling
                                             // function
}

// This function takes a String and returns one
fn takes_and_gives_back(a_string: String) -> String { // a_string comes into
                                                      // scope

    a_string  // a_string is returned and moves out to the calling function
}
\n
\n

What if we want to let a function use a value but not take ownership?
that’s reference

\n
\n

reference & borrow

    \n
  • & means reference (borrow but not own)， default immutable
    • \n
  • &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)
    • \n
  • Multiple mutable references can be created non-simultaneously by creating a new scope
    • \n
  • Cannot have mutable and immutable references at the same time
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    fn main() {
        let mut s = String::from("hello");
        {
            let s1 = &mut s;
        } // r1 goes out of scope here, so we can make a new reference with no problems.
        let s2 = &mut s;
    }

    fn main() {
        let mut s = String::from("hello");

        let r1 = &s; // no problem
        let r2 = &s; // no problem
        println!("{} and {}", r1, r2);
        // variables r1 and r2 will not be used after this point

        let r3 = &mut s; // no problem
        println!("{}", r3);

        println!{"{}",r1} // got problem with above mutable borrow
    }
    • \n
\n

reference as function arguments

1
2
3
4
5
6
7
8
9
10
11
12
fn main() {
    let s1 = String::from("hello");

    let len = calculate_length(&s1);

    println!("The length of '{}' is {}.", s1, len);
}

fn calculate_length(s: &String) -> usize { // s is a reference to a String
    s.len() 
}  // Here, s goes out of scope. But because it does not have ownership of what
   // it refers to, nothing happens.
\n

we pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used.
When functions have references as parameters instead of the actual values, we won’t need to return the values in order to give back ownership, because we never had ownership.
We call the action of creating a reference borrowing.

\n

mutable references

1
2
3
4
5
6
7
8
9
fn main() {
    let mut s = String::from("hello");

    change(&mut s);
}

fn change(some_string: &mut String) {
    some_string.push_str(", world");
}
\n\n

dangling references

    \n
  • A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else
    • \n
  • rust，The compiler can guarantee that there will never be dangling references
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let r = dangle();
    }
    fn dangle() -> &string {  // dangle returns a reference to a String
        let s = String::from("hello");  // s is a new String
        &s // we return a reference to the String, s
    } // Here, s goes out of scope, and is dropped. Its memory goes away.
    // Danger
    \nThe solution here is to return the String directly:
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let string = no_dangle();
    }

    fn no_dangle() -> String {
        let s = String::from("hello");
        s
    }
    \nThis works without any problems. Ownership is moved out, and nothing is deallocated.
    • \n
\n

slice

    \n
  • Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.
    • \n
\n

string slice

1
2
3
4
5
6
fn main() {
    let mut s = String::from("Hello world");

    let hello = &s[0..5]; 
    let world = &s[6..11];
}
\n

Rather than a reference to the entire String, hello is a reference to a portion of the String,
With Rust’s .. range syntax, if you want to start at index zero, you can drop the value before the two periods
By the same token, if your slice includes the last byte of the String, you can drop the trailing number.

\n
\n

Note: String slice range indices must occur at valid UTF-8 character boundaries. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.

\n
\n
1
2
3
4
5
6
7
8
9
fn first_word(s :&String) -> &str {
    let bytes = s.as_bytes();
    for(i, &item) in bytes.iter().enumerate() {
        if item == b' ' {
            return &s[..i];
        }
    }
    &s[..]
}
\n\n

String Literals Are Slices

1
2
3
fn main() {
let s = "Hello, world!";
}
\n

The type of s here is &str: it’s a slice pointing to that specific point of the binary.

\n

String Slices as Parameters

    \n
  • Pass &str as a parameter, you can receive parameters of type &String and &str at the same time
    1
    fn first_word(s: &String) -> &str
    \nequivalent to
    1
    fn first_word(s: &str) -> &str
    • \n
\n

other slices

array slice

\n
1
2
3
4
5
fn main() {
  let a = [1, 2, 3, 4, 5];
  let slice = &a[1..3];
  assert_eq!(slice, &[2, 3]);
}
","site":{"data":{}},"excerpt":"","more":"

ownership rule

    \n
  • Each value in Rust has an owner.
    • \n
  • There can only be one owner at a time.
    • \n
  • When the owner goes out of scope, the value will be dropped.
    • \n
\n

variable scope

1
2
3
4
5
6
fn main() {
    {                      // s is not valid here, it’s not yet declared
        let s = "hello";   // s is valid from this point forward
        // do stuff with s
    }                      // this scope is now over, and s is no longer valid
}
\n\n

Move

stack-only data: Copy trait

such as primitive type

\n
1
2
3
4
fn main() {
    let x = 5;
    let y = x;
}
\n

bind the value 5 to x; then make a copy of the value in x and bind it to y.

\n

for heap variable

1
2
3
4
fn main() {
    let s1 = String::from("hello");
    let s2 = s1;
}
\n

\"move-string\"
ptr, len, capacity is stored in stack, while string value is stored in heap
When we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap
\"move-string-2\"
similar to shallow copy

\n

Clone

1
2
3
4
5
6
fn main() {
    let s1 = String::from("hello");
    let s2 = s1.clone();

    println!("s1 = {}, s2 = {}", s1, s2);
}
\n

If we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.

\n

ownership and function

    \n
  • Passing a value to a function will result in a move or copy of ownership
    • \n
  • difference between “stack” and “heap” variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    fn main() {
        let s = String::from("hello");  // s comes into scope
        takes_ownership(s);             // s's value moves into the function...
                                        // ... and so is no longer valid here

        let x = 5;                      // x comes into scope

        makes_copy(x);                  // x would move into the function,
                                        // but i32 is Copy, so it's okay to still
                                        // use x afterward

    } // Here, x goes out of scope, then s. But because s's value was moved, nothing
      // special happens.

    fn takes_ownership(some_string: String) { // some_string comes into scope
        println!("{}", some_string);
    } // Here, some_string goes out of scope and `drop` is called. The backing
      // memory is freed.

    fn makes_copy(some_integer: i32) { // some_integer comes into scope
        println!("{}", some_integer);
    } // Here, some_integer goes out of scope. Nothing special happens.

    • \n
\n

return values and scope

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
fn main() {
    let s1 = gives_ownership();         // gives_ownership moves its return
                                        // value into s1

    let s2 = String::from("hello");     // s2 comes into scope

    let s3 = takes_and_gives_back(s2);  // s2 is moved into
                                        // takes_and_gives_back, which also
                                        // moves its return value into s3
} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing
  // happens. s1 goes out of scope and is dropped.

fn gives_ownership() -> String {             // gives_ownership will move its
                                             // return value into the function
                                             // that calls it

    let some_string = String::from("yours"); // some_string comes into scope

    some_string                              // some_string is returned and
                                             // moves out to the calling
                                             // function
}

// This function takes a String and returns one
fn takes_and_gives_back(a_string: String) -> String { // a_string comes into
                                                      // scope

    a_string  // a_string is returned and moves out to the calling function
}
\n
\n

What if we want to let a function use a value but not take ownership?
that’s reference

\n
\n

reference & borrow

    \n
  • & means reference (borrow but not own)， default immutable
    • \n
  • &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)
    • \n
  • Multiple mutable references can be created non-simultaneously by creating a new scope
    • \n
  • Cannot have mutable and immutable references at the same time
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    fn main() {
        let mut s = String::from("hello");
        {
            let s1 = &mut s;
        } // r1 goes out of scope here, so we can make a new reference with no problems.
        let s2 = &mut s;
    }

    fn main() {
        let mut s = String::from("hello");

        let r1 = &s; // no problem
        let r2 = &s; // no problem
        println!("{} and {}", r1, r2);
        // variables r1 and r2 will not be used after this point

        let r3 = &mut s; // no problem
        println!("{}", r3);

        println!{"{}",r1} // got problem with above mutable borrow
    }
    • \n
\n

reference as function arguments

1
2
3
4
5
6
7
8
9
10
11
12
fn main() {
    let s1 = String::from("hello");

    let len = calculate_length(&s1);

    println!("The length of '{}' is {}.", s1, len);
}

fn calculate_length(s: &String) -> usize { // s is a reference to a String
    s.len() 
}  // Here, s goes out of scope. But because it does not have ownership of what
   // it refers to, nothing happens.
\n

we pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used.
When functions have references as parameters instead of the actual values, we won’t need to return the values in order to give back ownership, because we never had ownership.
We call the action of creating a reference borrowing.

\n

mutable references

1
2
3
4
5
6
7
8
9
fn main() {
    let mut s = String::from("hello");

    change(&mut s);
}

fn change(some_string: &mut String) {
    some_string.push_str(", world");
}
\n\n

dangling references

    \n
  • A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else
    • \n
  • rust，The compiler can guarantee that there will never be dangling references
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let r = dangle();
    }
    fn dangle() -> &string {  // dangle returns a reference to a String
        let s = String::from("hello");  // s is a new String
        &s // we return a reference to the String, s
    } // Here, s goes out of scope, and is dropped. Its memory goes away.
    // Danger
    \nThe solution here is to return the String directly:
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let string = no_dangle();
    }

    fn no_dangle() -> String {
        let s = String::from("hello");
        s
    }
    \nThis works without any problems. Ownership is moved out, and nothing is deallocated.
    • \n
\n

slice

    \n
  • Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.
    • \n
\n

string slice

1
2
3
4
5
6
fn main() {
    let mut s = String::from("Hello world");

    let hello = &s[0..5]; 
    let world = &s[6..11];
}
\n

Rather than a reference to the entire String, hello is a reference to a portion of the String,
With Rust’s .. range syntax, if you want to start at index zero, you can drop the value before the two periods
By the same token, if your slice includes the last byte of the String, you can drop the trailing number.

\n
\n

Note: String slice range indices must occur at valid UTF-8 character boundaries. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.

\n
\n
1
2
3
4
5
6
7
8
9
fn first_word(s :&String) -> &str {
    let bytes = s.as_bytes();
    for(i, &item) in bytes.iter().enumerate() {
        if item == b' ' {
            return &s[..i];
        }
    }
    &s[..]
}
\n\n

String Literals Are Slices

1
2
3
fn main() {
let s = "Hello, world!";
}
\n

The type of s here is &str: it’s a slice pointing to that specific point of the binary.

\n

String Slices as Parameters

    \n
  • Pass &str as a parameter, you can receive parameters of type &String and &str at the same time
    1
    fn first_word(s: &String) -> &str
    \nequivalent to
    1
    fn first_word(s: &str) -> &str
    • \n
\n

other slices

array slice

\n
1
2
3
4
5
fn main() {
  let a = [1, 2, 3, 4, 5];
  let slice = &a[1..3];
  assert_eq!(slice, &[2, 3]);
}
"},{"title":"rust lifetime","date":"2022-10-18T13:33:26.000Z","_content":"\n## lifetime\n- Every reference in Rust has its own lifecycle\n- most of the time, Rust's lifetime is implicit and can be inferred\n- There are two types of life cycle: input life cycle and output life cycle\n- 'static is a special life cycle annotation\n### Example of lifetime out of scope\n```rust\nfn main() {\n let mut x;\n {\n let y = String::from(\"hello\");\n // x = y; // this is allowed\n x = &y; // not allowed. borrowed value (y) does not live long enough\n }\n println!(\"Str:{}\", x);\n}\n```\n\n### lifetime checker\nRust compiler's borrow checker to determine whether a borrow is legal\n```rust\n// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`\nfn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\nlet's find out why it is such case\n```rust\nfn main() {\n // variable to hold the result value\n let long_str; \n\n let x = \"abc\".to_string();\n {\n let y = \"bbccd\".to_string();\n long_str = longest(x.as_str(), y.as_str());\n }\n // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value\n println!(\"Longest str: {}\", long_str);\n}\n\n```\nHence, we need lifetime annotation `'`\n```rust\nfn longest<'a>(x:&'a str, y:&'a str) -> &'a str {\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\n\n### deeper understanding\n- When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters\n\n### Struct lifetime annotation\n```rust\nfn main() {\n let info = String::from(\"File not found.\");\n // 存放结果值的变量\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton<'a> {\n part: &'a str,\n}\n```\n- lifetime of the field `part` must be longer than struct\n\n### Lifetime Elision\nIn order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.\nElision rules are as follows:\n- Each elided lifetime in input position becomes a distinct lifetime parameter.\n- If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.\n- If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.\n- Otherwise, it is an error to elide an output lifetime.\n\n### struct lifetime annotation\n```rust\n\nfn main() {\n let info = String::from(\"File not found.\");\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton <'a>{\n part: &'a str,\n}\n\n// the first 'a is decraration, the second is usage\nimpl<'a> ImportantExcepiton <'a> {\n // in return value, 'a is omitted according to Lifetime Elision rule\n fn callname(&self ) -> &str{\n self.part\n }\n}\n```\n\n### 'static\n'static is a special lifetime that takes up the duration of the entire program, for example all string literals have a 'static lifetime","source":"_posts/rust/rust-04-lifetime.md","raw":"---\ntitle: rust lifetime\ndate: 2022-10-18 21:33:26\ntags: [rust]\n---\n\n## lifetime\n- Every reference in Rust has its own lifecycle\n- most of the time, Rust's lifetime is implicit and can be inferred\n- There are two types of life cycle: input life cycle and output life cycle\n- 'static is a special life cycle annotation\n### Example of lifetime out of scope\n```rust\nfn main() {\n let mut x;\n {\n let y = String::from(\"hello\");\n // x = y; // this is allowed\n x = &y; // not allowed. borrowed value (y) does not live long enough\n }\n println!(\"Str:{}\", x);\n}\n```\n\n### lifetime checker\nRust compiler's borrow checker to determine whether a borrow is legal\n```rust\n// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`\nfn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\nlet's find out why it is such case\n```rust\nfn main() {\n // variable to hold the result value\n let long_str; \n\n let x = \"abc\".to_string();\n {\n let y = \"bbccd\".to_string();\n long_str = longest(x.as_str(), y.as_str());\n }\n // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value\n println!(\"Longest str: {}\", long_str);\n}\n\n```\nHence, we need lifetime annotation `'`\n```rust\nfn longest<'a>(x:&'a str, y:&'a str) -> &'a str {\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\n\n### deeper understanding\n- When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters\n\n### Struct lifetime annotation\n```rust\nfn main() {\n let info = String::from(\"File not found.\");\n // 存放结果值的变量\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton<'a> {\n part: &'a str,\n}\n```\n- lifetime of the field `part` must be longer than struct\n\n### Lifetime Elision\nIn order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.\nElision rules are as follows:\n- Each elided lifetime in input position becomes a distinct lifetime parameter.\n- If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.\n- If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.\n- Otherwise, it is an error to elide an output lifetime.\n\n### struct lifetime annotation\n```rust\n\nfn main() {\n let info = String::from(\"File not found.\");\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton <'a>{\n part: &'a str,\n}\n\n// the first 'a is decraration, the second is usage\nimpl<'a> ImportantExcepiton <'a> {\n // in return value, 'a is omitted according to Lifetime Elision rule\n fn callname(&self ) -> &str{\n self.part\n }\n}\n```\n\n### 'static\n'static is a special lifetime that takes up the duration of the entire program, for example all string literals have a 'static lifetime","slug":"rust/rust-04-lifetime","published":1,"updated":"2023-05-01T14:08:49.387Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpr000f6hsj2rgy3f9c","content":"

lifetime

    \n
  • Every reference in Rust has its own lifecycle
    • \n
  • most of the time, Rust’s lifetime is implicit and can be inferred
    • \n
  • There are two types of life cycle: input life cycle and output life cycle
    • \n
  • ‘static is a special life cycle annotation
    • \n
\n

Example of lifetime out of scope

1
2
3
4
5
6
7
8
9
fn main() {
    let mut x;
    {
        let y = String::from("hello");
        // x = y; // this is allowed
        x = &y; // not allowed. borrowed value (y) does not live long enough
    }
    println!("Str:{}", x);
}
\n\n

lifetime checker

Rust compiler’s borrow checker to determine whether a borrow is legal

\n
1
2
3
4
5
6
7
8
// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`
fn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n

let’s find out why it is such case

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    // variable to hold the result value
    let long_str; 

    let x = "abc".to_string();
    {
        let y = "bbccd".to_string();
        long_str = longest(x.as_str(), y.as_str());
    }
    // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value
    println!("Longest str: {}", long_str);
}

\n

Hence, we need lifetime annotation '

\n
1
2
3
4
5
6
7
fn longest<'a>(x:&'a str, y:&'a str) -> &'a str {
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n\n

deeper understanding

    \n
  • When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters
    • \n
\n

Struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    let info = String::from("File not found.");
    // 存放结果值的变量
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton<'a> {
    part: &'a str,
}
\n
    \n
  • lifetime of the field part must be longer than struct
    • \n
\n

Lifetime Elision

In order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.
Elision rules are as follows:

\n
    \n
  • Each elided lifetime in input position becomes a distinct lifetime parameter.
    • \n
  • If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.
    • \n
  • If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.
    • \n
  • Otherwise, it is an error to elide an output lifetime.
    • \n
\n

struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

fn main() {
    let info = String::from("File not found.");
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton <'a>{
    part: &'a str,
}

// the first 'a is decraration, the second is usage
impl<'a> ImportantExcepiton <'a> {
    // in return value, 'a is omitted according to Lifetime Elision rule
    fn callname(&self ) -> &str{
        self.part
    }
}
\n\n

‘static

‘static is a special lifetime that takes up the duration of the entire program, for example all string literals have a ‘static lifetime

\n","site":{"data":{}},"excerpt":"","more":"

lifetime

    \n
  • Every reference in Rust has its own lifecycle
    • \n
  • most of the time, Rust’s lifetime is implicit and can be inferred
    • \n
  • There are two types of life cycle: input life cycle and output life cycle
    • \n
  • ‘static is a special life cycle annotation
    • \n
\n

Example of lifetime out of scope

1
2
3
4
5
6
7
8
9
fn main() {
    let mut x;
    {
        let y = String::from("hello");
        // x = y; // this is allowed
        x = &y; // not allowed. borrowed value (y) does not live long enough
    }
    println!("Str:{}", x);
}
\n\n

lifetime checker

Rust compiler’s borrow checker to determine whether a borrow is legal

\n
1
2
3
4
5
6
7
8
// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`
fn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n

let’s find out why it is such case

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    // variable to hold the result value
    let long_str; 

    let x = "abc".to_string();
    {
        let y = "bbccd".to_string();
        long_str = longest(x.as_str(), y.as_str());
    }
    // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value
    println!("Longest str: {}", long_str);
}

\n

Hence, we need lifetime annotation '

\n
1
2
3
4
5
6
7
fn longest<'a>(x:&'a str, y:&'a str) -> &'a str {
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n\n

deeper understanding

    \n
  • When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters
    • \n
\n

Struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    let info = String::from("File not found.");
    // 存放结果值的变量
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton<'a> {
    part: &'a str,
}
\n
    \n
  • lifetime of the field part must be longer than struct
    • \n
\n

Lifetime Elision

In order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.
Elision rules are as follows:

\n
    \n
  • Each elided lifetime in input position becomes a distinct lifetime parameter.
    • \n
  • If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.
    • \n
  • If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.
    • \n
  • Otherwise, it is an error to elide an output lifetime.
    • \n
\n

struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

fn main() {
    let info = String::from("File not found.");
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton <'a>{
    part: &'a str,
}

// the first 'a is decraration, the second is usage
impl<'a> ImportantExcepiton <'a> {
    // in return value, 'a is omitted according to Lifetime Elision rule
    fn callname(&self ) -> &str{
        self.part
    }
}
\n\n

‘static

‘static is a special lifetime that takes up the duration of the entire program, for example all string literals have a ‘static lifetime

\n"},{"title":"rust basics","date":"2022-10-04T07:55:04.000Z","_content":"\n## frequently used cmd\n```\nrustc [filename].rs\ncargo new [project_name]\ncargo build [--release]\ncargo run [--release]\ncargo check # check whether compile success, no executible output\n```\n\n## data type\n\n### integer\n- i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc\n- isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.\n- 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）\n\n| Number Literals | Example |\n| ----------- | ----------- |\n| Decimal | 98_222 |\n| Hex | 0xff |\n| Octal | 0o77 |\n| Binary | 0b1111_0000 |\n| Byte(u8 only) | b'A' |\n\n### Tuple\n- The length of Tuple is fixed, and the length cannot be changed once declared\n```rust\nfn main() {\n // tuple could be declared as mut\n let mut tuple_1 = (\"Hello\", 39, \"Years\");\n let tuple_2:(i32, &str ) = (1983, \"since.\");\n tuple_1.0 = \"Hi\";\n println!(\"{} {} {}\", tuple_1.0, tuple_1.1, tuple_1.2);\n // destructure\n let (a,b) = tuple_2;\n println!(\"{} {}\", a, b);\n}\n```\n\n### array\n- arrays in Rust have a fixed length.\n- Vector is similar to an array, it is provided by the standard library, and its length can be changed\n\n```rust\nfn main() {\n\n let arr_test:[u8; 3] = [1,2,3];\n println!(\"Number is {},{},{}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [\"I\",\"love\",\"you\"];\n println!(\"You said : {} {} {}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [1;3]; \n println!(\"Call Num : {}&{}&{}\", arr_test[0],arr_test[1],arr_test[2]);\n}\n```\n\n\n\n### String\n- Basic data types are stored on the stack, but the String type is stored on the heap\n```rust\nlet s = String::from(\"hello\");\n```\n- push_str(): append a str slice a string\n- push(): appends a single character to a String\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n}\n```\n- + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice\n- String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len\n- String iteration\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n\n for i in data.bytes() {\n ///\n }\n\n for i in data.chars() {\n ///\n }\n}\n```\n\n### Vector\n- Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.\n```rust\nfn main() {\n let vec: Vec = Vec::new();\n let vec2: Vec = vec![3,4，5] // create vector by macro\n for i in vec2 {\n println!(\"Vector value is : {}\", i);\n }\n}\n```\n\n### HashMap\n- HashMap is not preloaded, so it needs to be included `use std::collections::HashMap`\n```rust\nuse std::collections::HashMap;\nfn main() {\n let keys = vec![\"andy\".to_string(), \"cliff\".to_string()] ;\n let ages = vec![38, 26];\n let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();\n println!(\"{:?}\", map); /// print {\"andy\": 38, \"cliff\": 26}\n}\n```\n#### HashMap ownership\n- For types that implement the Copy trait (such as i32), the value will be copied into the HashMap\n- For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap\n- If a reference to a value is inserted into the HashMap, the value itself does not move\n\n#### HashMap iteration\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n println!(\"{:?}\", &map);\n for (k, v) in map {\n println!(\"{} age {}\", k, v);\n } /// cliff age 26\n /// andy age 36\n}\n```\n#### update\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n\n let result = map.entry(\"bob\".to_string());\n println!(\"{:?}\", result); /// Entry(VacantEntry(\"bob\"))\n\n let result = map.entry(\"andy\".to_string());\n println!(\"{:?}\", result); /// Entry(OccupiedEntry { key: \"andy\", value: 36, .. })\n\n map.entry(\"bob\".to_string()).or_insert(28);\n map.entry(\"cliff\".to_string()).or_insert(0);\n}\n```\n\n## control flow\n- if\n```rust\nfn main() {\n let condition = 1;\n let x = if condition == 1 { \"A\" } else { \"B\" };\n println!(\"Result x = {}\" , x) ;\n}\n```\n- loop\n```rust\nfn main() {\n let mut condition = 0;\n\n let result = 'outer: loop { // 'outer is label\n 'inner: loop {\n condition += 1;\n if 3 == condition {\n break 'outer 3 * condition; // break outer loop\n }\n }\n };\n println!(\"Loop result is : {}\", result); /// Loop result is : 9\n}\n\n```\n- rot\n```rust\nfn main() {\n let arr = [3,2,3];\n for num in arr.iter() {\n println!(\"For value is {}\", num);\n }\n}\n```\n\n## Range iterator\n- Range\n```rust\nfn main() {\n for number in (1..=3) {\n println!(\"Number A is {}\", number ); /// 1,2,3\n }\n \n for number in (1..=3).rev() { /// rev means reverse,\n println!(\"Number B is {}\", number ); /// 3,2,1\n }\n}\n\n```\n\n## struct\n- If struct is declared mutable then all fields in the instance are mutable\n### tuple struct\n```rust\nstruct Color(i32,i32,i32);\nlet black = Color(0,0,0);\n```\n### Unit-Like struct\n```rust\nstruct Man {};\n```\n### struct method\n```rust\n\nfn main() {\n let rec = Rectangle {\n width: 30,\n height: 50,\n };\n\n let result = rec.area(); \n println!(\"rectangle：{:?}，area is：{}\", rec, result);\n}\n\n\n#[derive(Debug)]\nstruct Rectangle {\n width: u32,\n height: u32,\n}\n\nimpl Rectangle {\n fn area(&self) -> u32{\n self.width * self.height\n }\n}\n\n```\n### associative func（similar to static method）\n- You can define a function that does not take `self` as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to `String::from()`\n```rust\nimpl Rectangle {\n fn create_square(width: u32) -> Rectangle {\n Rectangle {\n width,\n height: width,\n }\n }\n}\n```\n \n## enum\n```rust\nenum Ip {\n V4,\n V6,\n}\n\nenum IpAddr {\n V4(String),\n V6(String),\n}\n``` \n\n## match\n- match must exhaust all possibilities\n- If there are too many matchings, you can also use \"_\" for wildcarding, but note that \"_\" must be placed at the end\n```rust\nenum Color {\n Red,\n Yellow,\n Blue,\n}\nenum ColorWithVal {\n Red(u8,u8,u8),\n Yellow(u8,u8,u8),\n Blue(u8,u8,u8),\n}\nfn main(){\n let colour = Color::Blue;\n match colour {\n Color::Red => {\n println!(\"Red colour.\");\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n\n let colour = ColorWithVal::Red(222,111,22);\n match colour {\n ColorWithVal::Red(r,g,b) => {\n println!(\"Red colour. {},{},{}\", r,g,b);\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n}\n```\n\n## if let\n```rust\nfn main(){\n let colour = Color::Red(Some(222),Some(222),Some(222));\n\n if let Color::Red(r,g,b) = colour {\n println!(\"Red colour. {:?},{:?},{:?}\", r,g,b);\n } else {\n println!(\"Other colour.\");\n }\n}\n```\n\n## Result\n- Recoverable err via Result, non-recoverable via panic!\n- upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program\n- You can set panic = 'abort' in Cargo.toml to terminate the cleaning of the call stack\n```rust\n[profile.release]\npanic='abort'\n```\n- RUST_BACKTRACE = 1 prints detailed error messages in the stack\n```rust\nuse std::fs::File;\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => panic!(\"file not found {:?} \", error),\n };\n}\n```\n\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => {\n match error.kind() {\n ErrorKind::NotFound => {\n match File::create(\"hello.txt\") {\n Ok(file) => {\n file\n },\n Err(err) => {\n panic!(\"file create error:{:?}\", &err);\n },\n }\n },\n oe => panic!(\"other error {:?}\", oe),\n }\n } ,\n };\n}\n```\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let file = File::open(\"hello.txt\").unwrap_or_else(|err| {\n if err.kind() == ErrorKind::NotFound {\n File::create(\"hello.txt\").unwrap_or_else(|err|{\n panic!(\"error：{:?}\", err);\n })\n }else{\n panic!(\"other error：{:?}\", err);\n }\n });\n}\n```\n### unwrap && expect\n- If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.\n- expect can specify what the error message is, which is easier to debug\n\n### The question mark operator, ?\nWhen writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.\n```rust\nlet mut file = File::create(\"my_best_friends.txt\")?;\n```\n\n## generic\n```rust\n#[derive(Debug)]\nstruct Point {\n x : T,\n y : U,\n}\nimpl Point {\n fn mixup(self, other: Point) -> Point {\n Point{x: self.x , y: other.y, }\n }\n}\n```\n\n## trait\n### definition\n```rust\npub trait Summary {\n fn summarize(&self) -> String {\n \"... more\".to_string() /// default \n }\n}\npub struct Tweet {\n user_name :String,\n replay_count :u32,\n like_count :u32,\n}\nimpl Tweet {\n fn like(&mut self) {\n self.like_count += 1;\n }\n}\n\nimpl Summary for Tweet {\n fn summarize(&self) -> String {\n format!(\"{} like count :{} , replay count :{}\", &self.user_name, &self.replay_count, &self.like_count)\n }\n}\n```\n\n### trait as arguments\n```rust\nfn notify_msg (info: T) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: impl Summary + Display) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: T) \nwhere \n T: Summary + Display,\n{\n println!(\"summary : {}\", info.summarize() );\n println!(\"display implement info : {}\", info);\n}\n```\n### trait as return\n- impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported\n\n## references\n- [the rust programming language](https://doc.rust-lang.org/book/)\n- [mooc course](https://time.geekbang.org/course/intro/100060601?tab=catalog)\n- [bilibili tutorial](https://www.bilibili.com/video/BV1hp4y1k7SV?p=50&spm_id_from=pageDriver)\n- [jianshu notes](https://www.jianshu.com/p/30d917790298)","source":"_posts/rust/rust-02-basics.md","raw":"---\ntitle: rust basics\ndate: 2022-10-04 15:55:04\ntags: [rust]\n---\n\n## frequently used cmd\n```\nrustc [filename].rs\ncargo new [project_name]\ncargo build [--release]\ncargo run [--release]\ncargo check # check whether compile success, no executible output\n```\n\n## data type\n\n### integer\n- i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc\n- isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.\n- 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）\n\n| Number Literals | Example |\n| ----------- | ----------- |\n| Decimal | 98_222 |\n| Hex | 0xff |\n| Octal | 0o77 |\n| Binary | 0b1111_0000 |\n| Byte(u8 only) | b'A' |\n\n### Tuple\n- The length of Tuple is fixed, and the length cannot be changed once declared\n```rust\nfn main() {\n // tuple could be declared as mut\n let mut tuple_1 = (\"Hello\", 39, \"Years\");\n let tuple_2:(i32, &str ) = (1983, \"since.\");\n tuple_1.0 = \"Hi\";\n println!(\"{} {} {}\", tuple_1.0, tuple_1.1, tuple_1.2);\n // destructure\n let (a,b) = tuple_2;\n println!(\"{} {}\", a, b);\n}\n```\n\n### array\n- arrays in Rust have a fixed length.\n- Vector is similar to an array, it is provided by the standard library, and its length can be changed\n\n```rust\nfn main() {\n\n let arr_test:[u8; 3] = [1,2,3];\n println!(\"Number is {},{},{}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [\"I\",\"love\",\"you\"];\n println!(\"You said : {} {} {}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [1;3]; \n println!(\"Call Num : {}&{}&{}\", arr_test[0],arr_test[1],arr_test[2]);\n}\n```\n\n\n\n### String\n- Basic data types are stored on the stack, but the String type is stored on the heap\n```rust\nlet s = String::from(\"hello\");\n```\n- push_str(): append a str slice a string\n- push(): appends a single character to a String\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n}\n```\n- + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice\n- String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len\n- String iteration\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n\n for i in data.bytes() {\n ///\n }\n\n for i in data.chars() {\n ///\n }\n}\n```\n\n### Vector\n- Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.\n```rust\nfn main() {\n let vec: Vec = Vec::new();\n let vec2: Vec = vec![3,4，5] // create vector by macro\n for i in vec2 {\n println!(\"Vector value is : {}\", i);\n }\n}\n```\n\n### HashMap\n- HashMap is not preloaded, so it needs to be included `use std::collections::HashMap`\n```rust\nuse std::collections::HashMap;\nfn main() {\n let keys = vec![\"andy\".to_string(), \"cliff\".to_string()] ;\n let ages = vec![38, 26];\n let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();\n println!(\"{:?}\", map); /// print {\"andy\": 38, \"cliff\": 26}\n}\n```\n#### HashMap ownership\n- For types that implement the Copy trait (such as i32), the value will be copied into the HashMap\n- For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap\n- If a reference to a value is inserted into the HashMap, the value itself does not move\n\n#### HashMap iteration\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n println!(\"{:?}\", &map);\n for (k, v) in map {\n println!(\"{} age {}\", k, v);\n } /// cliff age 26\n /// andy age 36\n}\n```\n#### update\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n\n let result = map.entry(\"bob\".to_string());\n println!(\"{:?}\", result); /// Entry(VacantEntry(\"bob\"))\n\n let result = map.entry(\"andy\".to_string());\n println!(\"{:?}\", result); /// Entry(OccupiedEntry { key: \"andy\", value: 36, .. })\n\n map.entry(\"bob\".to_string()).or_insert(28);\n map.entry(\"cliff\".to_string()).or_insert(0);\n}\n```\n\n## control flow\n- if\n```rust\nfn main() {\n let condition = 1;\n let x = if condition == 1 { \"A\" } else { \"B\" };\n println!(\"Result x = {}\" , x) ;\n}\n```\n- loop\n```rust\nfn main() {\n let mut condition = 0;\n\n let result = 'outer: loop { // 'outer is label\n 'inner: loop {\n condition += 1;\n if 3 == condition {\n break 'outer 3 * condition; // break outer loop\n }\n }\n };\n println!(\"Loop result is : {}\", result); /// Loop result is : 9\n}\n\n```\n- rot\n```rust\nfn main() {\n let arr = [3,2,3];\n for num in arr.iter() {\n println!(\"For value is {}\", num);\n }\n}\n```\n\n## Range iterator\n- Range\n```rust\nfn main() {\n for number in (1..=3) {\n println!(\"Number A is {}\", number ); /// 1,2,3\n }\n \n for number in (1..=3).rev() { /// rev means reverse,\n println!(\"Number B is {}\", number ); /// 3,2,1\n }\n}\n\n```\n\n## struct\n- If struct is declared mutable then all fields in the instance are mutable\n### tuple struct\n```rust\nstruct Color(i32,i32,i32);\nlet black = Color(0,0,0);\n```\n### Unit-Like struct\n```rust\nstruct Man {};\n```\n### struct method\n```rust\n\nfn main() {\n let rec = Rectangle {\n width: 30,\n height: 50,\n };\n\n let result = rec.area(); \n println!(\"rectangle：{:?}，area is：{}\", rec, result);\n}\n\n\n#[derive(Debug)]\nstruct Rectangle {\n width: u32,\n height: u32,\n}\n\nimpl Rectangle {\n fn area(&self) -> u32{\n self.width * self.height\n }\n}\n\n```\n### associative func（similar to static method）\n- You can define a function that does not take `self` as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to `String::from()`\n```rust\nimpl Rectangle {\n fn create_square(width: u32) -> Rectangle {\n Rectangle {\n width,\n height: width,\n }\n }\n}\n```\n \n## enum\n```rust\nenum Ip {\n V4,\n V6,\n}\n\nenum IpAddr {\n V4(String),\n V6(String),\n}\n``` \n\n## match\n- match must exhaust all possibilities\n- If there are too many matchings, you can also use \"_\" for wildcarding, but note that \"_\" must be placed at the end\n```rust\nenum Color {\n Red,\n Yellow,\n Blue,\n}\nenum ColorWithVal {\n Red(u8,u8,u8),\n Yellow(u8,u8,u8),\n Blue(u8,u8,u8),\n}\nfn main(){\n let colour = Color::Blue;\n match colour {\n Color::Red => {\n println!(\"Red colour.\");\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n\n let colour = ColorWithVal::Red(222,111,22);\n match colour {\n ColorWithVal::Red(r,g,b) => {\n println!(\"Red colour. {},{},{}\", r,g,b);\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n}\n```\n\n## if let\n```rust\nfn main(){\n let colour = Color::Red(Some(222),Some(222),Some(222));\n\n if let Color::Red(r,g,b) = colour {\n println!(\"Red colour. {:?},{:?},{:?}\", r,g,b);\n } else {\n println!(\"Other colour.\");\n }\n}\n```\n\n## Result\n- Recoverable err via Result, non-recoverable via panic!\n- upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program\n- You can set panic = 'abort' in Cargo.toml to terminate the cleaning of the call stack\n```rust\n[profile.release]\npanic='abort'\n```\n- RUST_BACKTRACE = 1 prints detailed error messages in the stack\n```rust\nuse std::fs::File;\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => panic!(\"file not found {:?} \", error),\n };\n}\n```\n\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => {\n match error.kind() {\n ErrorKind::NotFound => {\n match File::create(\"hello.txt\") {\n Ok(file) => {\n file\n },\n Err(err) => {\n panic!(\"file create error:{:?}\", &err);\n },\n }\n },\n oe => panic!(\"other error {:?}\", oe),\n }\n } ,\n };\n}\n```\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let file = File::open(\"hello.txt\").unwrap_or_else(|err| {\n if err.kind() == ErrorKind::NotFound {\n File::create(\"hello.txt\").unwrap_or_else(|err|{\n panic!(\"error：{:?}\", err);\n })\n }else{\n panic!(\"other error：{:?}\", err);\n }\n });\n}\n```\n### unwrap && expect\n- If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.\n- expect can specify what the error message is, which is easier to debug\n\n### The question mark operator, ?\nWhen writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.\n```rust\nlet mut file = File::create(\"my_best_friends.txt\")?;\n```\n\n## generic\n```rust\n#[derive(Debug)]\nstruct Point {\n x : T,\n y : U,\n}\nimpl Point {\n fn mixup(self, other: Point) -> Point {\n Point{x: self.x , y: other.y, }\n }\n}\n```\n\n## trait\n### definition\n```rust\npub trait Summary {\n fn summarize(&self) -> String {\n \"... more\".to_string() /// default \n }\n}\npub struct Tweet {\n user_name :String,\n replay_count :u32,\n like_count :u32,\n}\nimpl Tweet {\n fn like(&mut self) {\n self.like_count += 1;\n }\n}\n\nimpl Summary for Tweet {\n fn summarize(&self) -> String {\n format!(\"{} like count :{} , replay count :{}\", &self.user_name, &self.replay_count, &self.like_count)\n }\n}\n```\n\n### trait as arguments\n```rust\nfn notify_msg (info: T) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: impl Summary + Display) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: T) \nwhere \n T: Summary + Display,\n{\n println!(\"summary : {}\", info.summarize() );\n println!(\"display implement info : {}\", info);\n}\n```\n### trait as return\n- impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported\n\n## references\n- [the rust programming language](https://doc.rust-lang.org/book/)\n- [mooc course](https://time.geekbang.org/course/intro/100060601?tab=catalog)\n- [bilibili tutorial](https://www.bilibili.com/video/BV1hp4y1k7SV?p=50&spm_id_from=pageDriver)\n- [jianshu notes](https://www.jianshu.com/p/30d917790298)","slug":"rust/rust-02-basics","published":1,"updated":"2023-05-01T09:07:13.124Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpr000h6hsj2ag6e8t0","content":"

frequently used cmd

1
2
3
4
5
rustc [filename].rs
cargo new [project_name]
cargo build [--release]
cargo run [--release]
cargo check # check whether compile success, no executible output
\n\n

data type

integer

    \n
  • i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc
    • \n
  • isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.
    • \n
  • 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）
    • \n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Number LiteralsExample
Decimal98_222
Hex0xff
Octal0o77
Binary0b1111_0000
Byte(u8 only)b’A’
\n

Tuple

    \n
  • The length of Tuple is fixed, and the length cannot be changed once declared
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
        // tuple could be declared as mut
        let mut tuple_1 = ("Hello", 39, "Years");
        let tuple_2:(i32, &str ) = (1983, "since.");
        tuple_1.0 = "Hi";
        println!("{} {} {}", tuple_1.0, tuple_1.1, tuple_1.2);
        // destructure
        let (a,b) = tuple_2;
        println!("{} {}", a, b);
    }
    • \n
\n

array

    \n
  • arrays in Rust have a fixed length.
    • \n
  • Vector is similar to an array, it is provided by the standard library, and its length can be changed
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
fn main() {

    let arr_test:[u8; 3] = [1,2,3];
    println!("Number is {},{},{}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = ["I","love","you"];
    println!("You said : {} {} {}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = [1;3]; 
    println!("Call Num : {}&{}&{}", arr_test[0],arr_test[1],arr_test[2]);
}
\n\n\n\n

String

    \n
  • Basic data types are stored on the stack, but the String type is stored on the heap
    1
    let s = String::from("hello");
    • \n
  • push_str(): append a str slice a string
    • \n
  • push(): appends a single character to a String
    1
    2
    3
    4
    5
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');
    }
    • \n
  • + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice
    • \n
  • String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len
    • \n
  • String iteration
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');

        for i in data.bytes() {
            ///
        }

        for i in data.chars() {
            ///
        }
    }
    • \n
\n

Vector

    \n
  • Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.
    1
    2
    3
    4
    5
    6
    7
    fn main() {
        let vec: Vec<u16> = Vec::new();
        let vec2: Vec<i32> = vec![3,4，5] // create vector by macro
        for i in vec2 {
            println!("Vector value is : {}", i);
        }
    }
    • \n
\n

HashMap

    \n
  • HashMap is not preloaded, so it needs to be included use std::collections::HashMap
    1
    2
    3
    4
    5
    6
    7
    use std::collections::HashMap;
    fn main() {
        let keys = vec!["andy".to_string(), "cliff".to_string()] ;
        let ages = vec![38, 26];
        let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();
        println!("{:?}", map); /// print {"andy": 38, "cliff": 26}
    }
    • \n
\n

HashMap ownership

    \n
  • For types that implement the Copy trait (such as i32), the value will be copied into the HashMap
    • \n
  • For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap
    • \n
  • If a reference to a value is inserted into the HashMap, the value itself does not move
    • \n
\n

HashMap iteration

1
2
3
4
5
6
7
8
9
10
11
12
13
14
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);
    println!("{:?}", &map);
    for (k, v) in map {
        println!("{} age {}", k, v);
    } /// cliff age 26
      /// andy age 36
}
\n

update

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);

    let result = map.entry("bob".to_string());
    println!("{:?}", result); /// Entry(VacantEntry("bob"))

    let result = map.entry("andy".to_string());
    println!("{:?}", result); /// Entry(OccupiedEntry { key: "andy", value: 36, .. })

    map.entry("bob".to_string()).or_insert(28);
    map.entry("cliff".to_string()).or_insert(0);
}
\n\n

control flow

    \n
  • if
    1
    2
    3
    4
    5
    fn main() {
        let condition = 1;
        let x = if condition == 1 { "A" } else { "B" };
        println!("Result x = {}" , x) ;
    }
    • \n
  • loop
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    fn main() {
        let mut condition = 0;

        let result = 'outer: loop {  // 'outer is label
            'inner: loop {
                condition += 1;
                if 3 == condition {
                    break 'outer 3 * condition; // break outer loop
                }
            }
        };
        println!("Loop result is : {}", result); /// Loop result is : 9
    }

    • \n
  • rot
    1
    2
    3
    4
    5
    6
    fn main() {
        let arr = [3,2,3];
        for num in arr.iter() {
            println!("For value is {}", num);
        }
    }
    • \n
\n

Range iterator

    \n
  • Range
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
         for number in (1..=3) {
            println!("Number A is {}", number ); /// 1,2,3
        }
     
        for number in (1..=3).rev() { /// rev means reverse,
            println!("Number B is {}", number ); /// 3,2,1
        }
    }

    • \n
\n

struct

    \n
  • If struct is declared mutable then all fields in the instance are mutable
    • \n
\n

tuple struct

1
2
struct Color(i32,i32,i32);
let black = Color(0,0,0);
\n

Unit-Like struct

1
struct Man {};
\n

struct method

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

fn main() {
    let rec = Rectangle {
        width: 30,
        height: 50,
    };

    let result = rec.area(); 
    println!("rectangle：{:?}，area is：{}", rec, result);
}


#[derive(Debug)]
struct Rectangle {
    width: u32,
    height: u32,
}

impl Rectangle {
    fn area(&self) -> u32{
        self.width * self.height
    }
}

\n

associative func（similar to static method）

    \n
  • You can define a function that does not take self as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to String::from()
    1
    2
    3
    4
    5
    6
    7
    8
    impl Rectangle {
        fn create_square(width: u32) -> Rectangle {
            Rectangle {
                width,
                height: width,
            }
        }
    }
    • \n
\n

enum

1
2
3
4
5
6
7
8
9
enum Ip {
    V4,
    V6,
}

enum IpAddr {
    V4(String),
    V6(String),
}
\n\n

match

    \n
  • match must exhaust all possibilities
    • \n
  • If there are too many matchings, you can also use ““ for wildcarding, but note that ““ must be placed at the end
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    enum Color {
        Red,
        Yellow,
        Blue,
    }
    enum ColorWithVal {
        Red(u8,u8,u8),
        Yellow(u8,u8,u8),
        Blue(u8,u8,u8),
    }
    fn main(){
        let colour = Color::Blue;
        match colour {
            Color::Red => {
                println!("Red colour.");
            },
            _ => {
                println!("Other colour.");
            }
        }

        let colour = ColorWithVal::Red(222,111,22);
        match colour {
            ColorWithVal::Red(r,g,b) => {
                println!("Red colour. {},{},{}", r,g,b);
            },
            _ => {
                println!("Other colour.");
            }
        }
    }
    • \n
\n

if let

1
2
3
4
5
6
7
8
9
fn main(){
    let colour = Color::Red(Some(222),Some(222),Some(222));

    if let Color::Red(r,g,b) = colour {
        println!("Red colour. {:?},{:?},{:?}", r,g,b);
    } else {
        println!("Other colour.");
    }
}
\n\n

Result<T,E>

    \n
  • Recoverable err via Result<T,E>, non-recoverable via panic!
    • \n
  • upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program
    • \n
  • You can set panic = ‘abort’ in Cargo.toml to terminate the cleaning of the call stack
    1
    2
    [profile.release]
    panic='abort'
    • \n
  • RUST_BACKTRACE = 1 prints detailed error messages in the stack
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::fs::File;
    fn main() { 
        let fp = File::open("hello.txt");
        let file = match fp {
            Ok(file)=> {
                file
            },
            Err(error) => panic!("file not found {:?} ", error),
        };
    }
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
use std::{fs::File, io::ErrorKind};
fn main() { 
    let fp = File::open("hello.txt");
    let file = match fp {
        Ok(file)=> {
            file
        },
        Err(error) => {
            match error.kind() {
                ErrorKind::NotFound => {
                    match File::create("hello.txt") {
                        Ok(file) => {
                            file
                        },
                        Err(err) => {
                            panic!("file create error:{:?}", &err);
                        },
                    }
                },
                oe => panic!("other error {:?}", oe),
            }
        } ,
    };
}
\n
1
2
3
4
5
6
7
8
9
10
11
12
use std::{fs::File, io::ErrorKind};
fn main() { 
    let file = File::open("hello.txt").unwrap_or_else(|err| {
        if err.kind() == ErrorKind::NotFound {
            File::create("hello.txt").unwrap_or_else(|err|{
                panic!("error：{:?}", err);
            })
        }else{
            panic!("other error：{:?}", err);
        }
    });
}
\n

unwrap && expect

    \n
  • If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.
    • \n
  • expect can specify what the error message is, which is easier to debug
    • \n
\n

The question mark operator, ?

When writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.

\n
1
let mut file = File::create("my_best_friends.txt")?;
\n\n

generic

1
2
3
4
5
6
7
8
9
10
#[derive(Debug)]
struct Point<T, U>  {
   x : T,
   y : U,
}
impl <T, U> Point<T, U> {
    fn mixup<V, W>(self, other: Point<V, W>) -> Point<T, W> {
        Point{x: self.x , y: other.y, }
    }
}
\n\n

trait

definition

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
pub trait Summary {
    fn summarize(&self) -> String {
         "... more".to_string() /// default 
    }
}
pub struct Tweet {
    user_name :String,
    replay_count :u32,
    like_count :u32,
}
impl Tweet {
    fn like(&mut self) {
        self.like_count += 1;
    }
}

impl Summary for Tweet {
    fn summarize(&self) -> String {
        format!("{} like count :{} , replay count :{}", &self.user_name, &self.replay_count, &self.like_count)
    }
}
\n\n

trait as arguments

1
2
3
4
5
6
7
8
9
10
11
12
13
fn notify_msg <T:Summary> (info: T) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg (info: impl Summary + Display) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg <T> (info: T) 
where 
    T: Summary + Display,
{
    println!("summary : {}", info.summarize() );
    println!("display implement info : {}", info);
}
\n

trait as return

    \n
  • impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported
    • \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"

frequently used cmd

1
2
3
4
5
rustc [filename].rs
cargo new [project_name]
cargo build [--release]
cargo run [--release]
cargo check # check whether compile success, no executible output
\n\n

data type

integer

    \n
  • i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc
    • \n
  • isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.
    • \n
  • 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）
    • \n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Number LiteralsExample
Decimal98_222
Hex0xff
Octal0o77
Binary0b1111_0000
Byte(u8 only)b’A’
\n

Tuple

    \n
  • The length of Tuple is fixed, and the length cannot be changed once declared
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
        // tuple could be declared as mut
        let mut tuple_1 = ("Hello", 39, "Years");
        let tuple_2:(i32, &str ) = (1983, "since.");
        tuple_1.0 = "Hi";
        println!("{} {} {}", tuple_1.0, tuple_1.1, tuple_1.2);
        // destructure
        let (a,b) = tuple_2;
        println!("{} {}", a, b);
    }
    • \n
\n

array

    \n
  • arrays in Rust have a fixed length.
    • \n
  • Vector is similar to an array, it is provided by the standard library, and its length can be changed
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
fn main() {

    let arr_test:[u8; 3] = [1,2,3];
    println!("Number is {},{},{}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = ["I","love","you"];
    println!("You said : {} {} {}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = [1;3]; 
    println!("Call Num : {}&{}&{}", arr_test[0],arr_test[1],arr_test[2]);
}
\n\n\n\n

String

    \n
  • Basic data types are stored on the stack, but the String type is stored on the heap
    1
    let s = String::from("hello");
    • \n
  • push_str(): append a str slice a string
    • \n
  • push(): appends a single character to a String
    1
    2
    3
    4
    5
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');
    }
    • \n
  • + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice
    • \n
  • String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len
    • \n
  • String iteration
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');

        for i in data.bytes() {
            ///
        }

        for i in data.chars() {
            ///
        }
    }
    • \n
\n

Vector

    \n
  • Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.
    1
    2
    3
    4
    5
    6
    7
    fn main() {
        let vec: Vec<u16> = Vec::new();
        let vec2: Vec<i32> = vec![3,4，5] // create vector by macro
        for i in vec2 {
            println!("Vector value is : {}", i);
        }
    }
    • \n
\n

HashMap

    \n
  • HashMap is not preloaded, so it needs to be included use std::collections::HashMap
    1
    2
    3
    4
    5
    6
    7
    use std::collections::HashMap;
    fn main() {
        let keys = vec!["andy".to_string(), "cliff".to_string()] ;
        let ages = vec![38, 26];
        let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();
        println!("{:?}", map); /// print {"andy": 38, "cliff": 26}
    }
    • \n
\n

HashMap ownership

    \n
  • For types that implement the Copy trait (such as i32), the value will be copied into the HashMap
    • \n
  • For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap
    • \n
  • If a reference to a value is inserted into the HashMap, the value itself does not move
    • \n
\n

HashMap iteration

1
2
3
4
5
6
7
8
9
10
11
12
13
14
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);
    println!("{:?}", &map);
    for (k, v) in map {
        println!("{} age {}", k, v);
    } /// cliff age 26
      /// andy age 36
}
\n

update

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);

    let result = map.entry("bob".to_string());
    println!("{:?}", result); /// Entry(VacantEntry("bob"))

    let result = map.entry("andy".to_string());
    println!("{:?}", result); /// Entry(OccupiedEntry { key: "andy", value: 36, .. })

    map.entry("bob".to_string()).or_insert(28);
    map.entry("cliff".to_string()).or_insert(0);
}
\n\n

control flow

    \n
  • if
    1
    2
    3
    4
    5
    fn main() {
        let condition = 1;
        let x = if condition == 1 { "A" } else { "B" };
        println!("Result x = {}" , x) ;
    }
    • \n
  • loop
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    fn main() {
        let mut condition = 0;

        let result = 'outer: loop {  // 'outer is label
            'inner: loop {
                condition += 1;
                if 3 == condition {
                    break 'outer 3 * condition; // break outer loop
                }
            }
        };
        println!("Loop result is : {}", result); /// Loop result is : 9
    }

    • \n
  • rot
    1
    2
    3
    4
    5
    6
    fn main() {
        let arr = [3,2,3];
        for num in arr.iter() {
            println!("For value is {}", num);
        }
    }
    • \n
\n

Range iterator

    \n
  • Range
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
         for number in (1..=3) {
            println!("Number A is {}", number ); /// 1,2,3
        }
     
        for number in (1..=3).rev() { /// rev means reverse,
            println!("Number B is {}", number ); /// 3,2,1
        }
    }

    • \n
\n

struct

    \n
  • If struct is declared mutable then all fields in the instance are mutable
    • \n
\n

tuple struct

1
2
struct Color(i32,i32,i32);
let black = Color(0,0,0);
\n

Unit-Like struct

1
struct Man {};
\n

struct method

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

fn main() {
    let rec = Rectangle {
        width: 30,
        height: 50,
    };

    let result = rec.area(); 
    println!("rectangle：{:?}，area is：{}", rec, result);
}


#[derive(Debug)]
struct Rectangle {
    width: u32,
    height: u32,
}

impl Rectangle {
    fn area(&self) -> u32{
        self.width * self.height
    }
}

\n

associative func（similar to static method）

    \n
  • You can define a function that does not take self as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to String::from()
    1
    2
    3
    4
    5
    6
    7
    8
    impl Rectangle {
        fn create_square(width: u32) -> Rectangle {
            Rectangle {
                width,
                height: width,
            }
        }
    }
    • \n
\n

enum

1
2
3
4
5
6
7
8
9
enum Ip {
    V4,
    V6,
}

enum IpAddr {
    V4(String),
    V6(String),
}
\n\n

match

    \n
  • match must exhaust all possibilities
    • \n
  • If there are too many matchings, you can also use ““ for wildcarding, but note that ““ must be placed at the end
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    enum Color {
        Red,
        Yellow,
        Blue,
    }
    enum ColorWithVal {
        Red(u8,u8,u8),
        Yellow(u8,u8,u8),
        Blue(u8,u8,u8),
    }
    fn main(){
        let colour = Color::Blue;
        match colour {
            Color::Red => {
                println!("Red colour.");
            },
            _ => {
                println!("Other colour.");
            }
        }

        let colour = ColorWithVal::Red(222,111,22);
        match colour {
            ColorWithVal::Red(r,g,b) => {
                println!("Red colour. {},{},{}", r,g,b);
            },
            _ => {
                println!("Other colour.");
            }
        }
    }
    • \n
\n

if let

1
2
3
4
5
6
7
8
9
fn main(){
    let colour = Color::Red(Some(222),Some(222),Some(222));

    if let Color::Red(r,g,b) = colour {
        println!("Red colour. {:?},{:?},{:?}", r,g,b);
    } else {
        println!("Other colour.");
    }
}
\n\n

Result<T,E>

    \n
  • Recoverable err via Result<T,E>, non-recoverable via panic!
    • \n
  • upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program
    • \n
  • You can set panic = ‘abort’ in Cargo.toml to terminate the cleaning of the call stack
    1
    2
    [profile.release]
    panic='abort'
    • \n
  • RUST_BACKTRACE = 1 prints detailed error messages in the stack
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::fs::File;
    fn main() { 
        let fp = File::open("hello.txt");
        let file = match fp {
            Ok(file)=> {
                file
            },
            Err(error) => panic!("file not found {:?} ", error),
        };
    }
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
use std::{fs::File, io::ErrorKind};
fn main() { 
    let fp = File::open("hello.txt");
    let file = match fp {
        Ok(file)=> {
            file
        },
        Err(error) => {
            match error.kind() {
                ErrorKind::NotFound => {
                    match File::create("hello.txt") {
                        Ok(file) => {
                            file
                        },
                        Err(err) => {
                            panic!("file create error:{:?}", &err);
                        },
                    }
                },
                oe => panic!("other error {:?}", oe),
            }
        } ,
    };
}
\n
1
2
3
4
5
6
7
8
9
10
11
12
use std::{fs::File, io::ErrorKind};
fn main() { 
    let file = File::open("hello.txt").unwrap_or_else(|err| {
        if err.kind() == ErrorKind::NotFound {
            File::create("hello.txt").unwrap_or_else(|err|{
                panic!("error：{:?}", err);
            })
        }else{
            panic!("other error：{:?}", err);
        }
    });
}
\n

unwrap && expect

    \n
  • If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.
    • \n
  • expect can specify what the error message is, which is easier to debug
    • \n
\n

The question mark operator, ?

When writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.

\n
1
let mut file = File::create("my_best_friends.txt")?;
\n\n

generic

1
2
3
4
5
6
7
8
9
10
#[derive(Debug)]
struct Point<T, U>  {
   x : T,
   y : U,
}
impl <T, U> Point<T, U> {
    fn mixup<V, W>(self, other: Point<V, W>) -> Point<T, W> {
        Point{x: self.x , y: other.y, }
    }
}
\n\n

trait

definition

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
pub trait Summary {
    fn summarize(&self) -> String {
         "... more".to_string() /// default 
    }
}
pub struct Tweet {
    user_name :String,
    replay_count :u32,
    like_count :u32,
}
impl Tweet {
    fn like(&mut self) {
        self.like_count += 1;
    }
}

impl Summary for Tweet {
    fn summarize(&self) -> String {
        format!("{} like count :{} , replay count :{}", &self.user_name, &self.replay_count, &self.like_count)
    }
}
\n\n

trait as arguments

1
2
3
4
5
6
7
8
9
10
11
12
13
fn notify_msg <T:Summary> (info: T) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg (info: impl Summary + Display) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg <T> (info: T) 
where 
    T: Summary + Display,
{
    println!("summary : {}", info.summarize() );
    println!("display implement info : {}", info);
}
\n

trait as return

    \n
  • impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported
    • \n
\n

references

\n"},{"title":"rust smart pointer","date":"2022-10-30T03:00:38.000Z","_content":"\n## Overview\nThe most common kind of pointer in Rust is a reference (borrow but not own)\nSmart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.\nreferences are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.\n\n### smart pointers example\n- String\n- Vec\nBoth these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).\n\n### Deref & Drop\nSmart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the `Deref` and `Drop` traits.\n- The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers. \n- The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.\n\n### the most common smart pointers in the standard library:\n- `Box` for allocating values on the heap\n- `Rc`, a reference counting type that enables multiple ownership\n- `Ref` and `RefMut`, accessed through `RefCell`, a type that enforces the borrowing rules at runtime instead of compile time\n\n## Box\n### when to use\n- When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)\n- When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so\n- When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type\n\n\n### enabling recursive types with Box\nAt compile time, Rust needs to know how much space a type takes up\nOne type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address\n\n```rust\nenum List {\n Cons(i32, List),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nfn main() {\n let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size\n}\n```\n\nuse Box\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));\n}\n```\n\n## Treating Smart Pointers Like Regular References with the Deref Trait\nImplementing the Deref trait allows you to customize the behavior of the dereference operator, `*`\nBy implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.\n\n### Defining Our Own Smart Pointer\n```rust\nuse std::ops::Deref;\n\nstruct MyBox(T); // The MyBox type is a tuple struct with one element of type T\n\nimpl MyBox {\n fn new(x: T) -> MyBox {\n MyBox(x)\n }\n}\n\nimpl Deref for MyBox {\n type Target = T;\n\n fn deref(&self) -> &Self::Target {\n &self.0\n }\n}\n\nfn main() {\n let x = 5;\n let y = MyBox::new(x);\n\n assert_eq!(5, x);\n assert_eq!(5, *y);\n}\n```\n- We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator. \n- behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method\n- The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.\n\n### Implicit Deref Coercions with Functions and Methods\nDeref coercion is a convenience that Rust performs on arguments to functions and methods. \nDeref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str\nA sequence of calls to the deref method converts the type we provided into the type the parameter needs.\n\n```rust\nfn hello(name: &str) {\n println!(\"Hello, {}!\", name);\n}\n\nfn main() {\n let m = MyBox::new(String::from(\"Rust\"));\n hello(\"rust\"); // ok\n hello(&m); // also ok\n hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello\n}\n```\nHere we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.\n\n### How Deref Coercion Interacts with Mutability\nSimilar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.\n\nRust does deref coercion when it finds types and trait implementations in three cases:\n- From &T to &U when T: Deref\n- From &mut T to &mut U when T: DerefMut\n- From &mut T to &U when T: Deref\n\n## Running Code on Cleanup with the Drop Trait\nDrop, which lets you customize what happens when a value is about to go out of scope. \n\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"my stuff\"),\n };\n c.drop(); // not allowed\n let d = CustomSmartPointer {\n data: String::from(\"other stuff\"),\n };\n println!(\"CustomSmartPointers created.\");\n}\n\n```\n### Dropping a Value Early with std::mem::drop\nstd::mem::drop is in prelude, can use directly\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"some data\"),\n };\n println!(\"CustomSmartPointer created.\");\n drop(c); // ok\n println!(\"CustomSmartPointer dropped before the end of main.\");\n} // c goes out of scope, will occur double drop\n\n```\n\n\n## Rc: the Reference Counted Smart Pointer\nIn the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners. \ntype keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.\n\nWe use the `Rc` type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.\n\nRc is only for use in single-threaded scenarios\n\n### using Rc to share data\n\n![rc](/images/rust/pointers/rc.png)\nimplement use box will not work, as below\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));\n let b = Cons(3, Box::new(a)); // value moved here\n let c = Cons(4, Box::new(a)); // value used here after move\n}\n```\nuse Rc\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data\n let c = Cons(4, Rc::clone(&a));\n}\n```\n\n\nWe could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.\n\n### Cloning an Rc Increases the Reference Count\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n println!(\"count after creating a = {}\", Rc::strong_count(&a)); // 1\n let b = Cons(3, Rc::clone(&a));\n println!(\"count after creating b = {}\", Rc::strong_count(&a)); // 2\n {\n let c = Cons(4, Rc::clone(&a));\n println!(\"count after creating c = {}\", Rc::strong_count(&a)); // 3\n }\n println!(\"count after c goes out of scope = {}\", Rc::strong_count(&a)); // 2\n}\n```\nWhat we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.\n\n## RefCell and interior mutability pattern\nInterior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.\nTo mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.\nWe can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that. \nThe unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.\n\n### Enforcing Borrowing Rules at Runtime with RefCell\nUnlike Rc, the RefCell type represents single ownership over the data it holds.\n\nrecall borrowing rules\n- At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.\n- References must always be valid.\nWith references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime. \nBecause RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.\n\n### Interior Mutability: A Mutable Borrow to an Immutable Value\n```rust\nfn main() {\n let x = 5;\n let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable\n}\n```\n\n### A Use Case for Interior Mutability: Mock Objects\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n```\na problematic usage\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n struct MockMessenger {\n sent_messages: Vec,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: vec![],\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.len(), 1);\n }\n}\n\n```\nWe can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition\n\nThis is a situation in which interior mutability can help! \n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::cell::RefCell;\n\n struct MockMessenger {\n sent_messages: RefCell>,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: RefCell::new(vec![]),\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell> in self.sent_messages to get a mutable reference to the value inside the RefCell>\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n // --snip--\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); // call borrow on the RefCell> to get an immutable reference to the vector.\n }\n}\n```\n\n### Keeping Track of Borrows at Runtime with RefCell\nWhen creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. **The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut**. Both types implement Deref, so we can treat them like regular references.\n\nThe RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime. \n```rust\nimpl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n let mut one_borrow = self.sent_messages.borrow_mut();\n let mut two_borrow = self.sent_messages.borrow_mut();\n\n one_borrow.push(String::from(message));\n two_borrow.push(String::from(message));\n }\n }\n```\nWhen we run the tests for our library, the code in will compile without any errors, but the test will fail\nthread 'main' panicked at 'already borrowed\n\n### Having Multiple Owners of Mutable Data by Combining Rc and RefCell\nA common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!\n```rust\n#[derive(Debug)]\nenum List {\n Cons(Rc>, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\nfn main() {\n let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc> and store it in a variable named value so we can access it directly later.\n\n let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); // Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc so when we create lists b and c, they can both refer to a\n\n let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));\n let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));\n\n *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc to the inner RefCell value. The borrow_mut method returns a RefMut smart pointer, and we use the dereference operator on it and change the inner value.\n\n println!(\"a after = {:?}\", a);\n println!(\"b after = {:?}\", b);\n println!(\"c after = {:?}\", c);\n}\n```\n\n**The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;** \n\n## Reference Cycles Can Leak Memory\nRust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).\n\n### Creating a Reference Cycle\n```rust\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>), // The second element in the Cons variant is now RefCell>, meaning that we want to modify which List value a Cons variant is pointing to. \n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n use crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>),\n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> {\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));\n\n println!(\"a initial rc count = {}\", Rc::strong_count(&a));\n println!(\"a next item = {:?}\", a.tail());\n\n let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a\n\n println!(\"a rc count after b creation = {}\", Rc::strong_count(&a));\n println!(\"b initial rc count = {}\", Rc::strong_count(&b));\n println!(\"b next item = {:?}\", b.tail());\n\n if let Some(link) = a.tail() {\n *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle\n }\n\n println!(\"b rc count after changing a = {}\", Rc::strong_count(&b));\n println!(\"a rc count after changing a = {}\", Rc::strong_count(&a));\n\n // Uncomment the next line to see that we have a cycle;\n // it will overflow the stack\n // println!(\"a next item = {:?}\", a.tail());\n} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc instance from 2 to 1. The memory that Rc has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc instance still refers to it. The memory allocated to the list will remain uncollected forever.\n```\n![cycle-ref](/images/rust/pointers/cycle.ref.png)\n\n### Preventing Reference Cycles: Turning an Rc into a Weak\nSo far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.\n\nStrong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.\n\nBecause the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped. \n\n\n### Creating a Tree Data Structure: a Node with Child Nodes\n```rust\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.\n children: RefCell>>,\n}\n\nfn main() {\n\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. \n\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. \n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak reference to branch from the Rc in branch.\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n\n \n}\n```\n\n### Visualizing Changes to strong_count and weak_count\n```rust\nuse std::cell::RefCell;\nuse std::rc::{Rc, Weak};\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>,\n children: RefCell>>,\n}\n\nfn main() {\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n\n {\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]),\n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch);\n\n println!(\n \"branch strong = {}, weak = {}\",\n Rc::strong_count(&branch),\n Rc::weak_count(&branch),\n );\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n }\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n}\n```","source":"_posts/rust/rust-06-smart-pointer.md","raw":"---\ntitle: rust smart pointer\ndate: 2022-10-30 11:00:38\ntags: [rust]\n---\n\n## Overview\nThe most common kind of pointer in Rust is a reference (borrow but not own)\nSmart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.\nreferences are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.\n\n### smart pointers example\n- String\n- Vec\nBoth these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).\n\n### Deref & Drop\nSmart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the `Deref` and `Drop` traits.\n- The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers. \n- The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.\n\n### the most common smart pointers in the standard library:\n- `Box` for allocating values on the heap\n- `Rc`, a reference counting type that enables multiple ownership\n- `Ref` and `RefMut`, accessed through `RefCell`, a type that enforces the borrowing rules at runtime instead of compile time\n\n## Box\n### when to use\n- When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)\n- When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so\n- When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type\n\n\n### enabling recursive types with Box\nAt compile time, Rust needs to know how much space a type takes up\nOne type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address\n\n```rust\nenum List {\n Cons(i32, List),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nfn main() {\n let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size\n}\n```\n\nuse Box\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));\n}\n```\n\n## Treating Smart Pointers Like Regular References with the Deref Trait\nImplementing the Deref trait allows you to customize the behavior of the dereference operator, `*`\nBy implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.\n\n### Defining Our Own Smart Pointer\n```rust\nuse std::ops::Deref;\n\nstruct MyBox(T); // The MyBox type is a tuple struct with one element of type T\n\nimpl MyBox {\n fn new(x: T) -> MyBox {\n MyBox(x)\n }\n}\n\nimpl Deref for MyBox {\n type Target = T;\n\n fn deref(&self) -> &Self::Target {\n &self.0\n }\n}\n\nfn main() {\n let x = 5;\n let y = MyBox::new(x);\n\n assert_eq!(5, x);\n assert_eq!(5, *y);\n}\n```\n- We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator. \n- behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method\n- The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.\n\n### Implicit Deref Coercions with Functions and Methods\nDeref coercion is a convenience that Rust performs on arguments to functions and methods. \nDeref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str\nA sequence of calls to the deref method converts the type we provided into the type the parameter needs.\n\n```rust\nfn hello(name: &str) {\n println!(\"Hello, {}!\", name);\n}\n\nfn main() {\n let m = MyBox::new(String::from(\"Rust\"));\n hello(\"rust\"); // ok\n hello(&m); // also ok\n hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello\n}\n```\nHere we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.\n\n### How Deref Coercion Interacts with Mutability\nSimilar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.\n\nRust does deref coercion when it finds types and trait implementations in three cases:\n- From &T to &U when T: Deref\n- From &mut T to &mut U when T: DerefMut\n- From &mut T to &U when T: Deref\n\n## Running Code on Cleanup with the Drop Trait\nDrop, which lets you customize what happens when a value is about to go out of scope. \n\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"my stuff\"),\n };\n c.drop(); // not allowed\n let d = CustomSmartPointer {\n data: String::from(\"other stuff\"),\n };\n println!(\"CustomSmartPointers created.\");\n}\n\n```\n### Dropping a Value Early with std::mem::drop\nstd::mem::drop is in prelude, can use directly\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"some data\"),\n };\n println!(\"CustomSmartPointer created.\");\n drop(c); // ok\n println!(\"CustomSmartPointer dropped before the end of main.\");\n} // c goes out of scope, will occur double drop\n\n```\n\n\n## Rc: the Reference Counted Smart Pointer\nIn the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners. \ntype keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.\n\nWe use the `Rc` type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.\n\nRc is only for use in single-threaded scenarios\n\n### using Rc to share data\n\n![rc](/images/rust/pointers/rc.png)\nimplement use box will not work, as below\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));\n let b = Cons(3, Box::new(a)); // value moved here\n let c = Cons(4, Box::new(a)); // value used here after move\n}\n```\nuse Rc\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data\n let c = Cons(4, Rc::clone(&a));\n}\n```\n\n\nWe could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.\n\n### Cloning an Rc Increases the Reference Count\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n println!(\"count after creating a = {}\", Rc::strong_count(&a)); // 1\n let b = Cons(3, Rc::clone(&a));\n println!(\"count after creating b = {}\", Rc::strong_count(&a)); // 2\n {\n let c = Cons(4, Rc::clone(&a));\n println!(\"count after creating c = {}\", Rc::strong_count(&a)); // 3\n }\n println!(\"count after c goes out of scope = {}\", Rc::strong_count(&a)); // 2\n}\n```\nWhat we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.\n\n## RefCell and interior mutability pattern\nInterior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.\nTo mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.\nWe can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that. \nThe unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.\n\n### Enforcing Borrowing Rules at Runtime with RefCell\nUnlike Rc, the RefCell type represents single ownership over the data it holds.\n\nrecall borrowing rules\n- At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.\n- References must always be valid.\nWith references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime. \nBecause RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.\n\n### Interior Mutability: A Mutable Borrow to an Immutable Value\n```rust\nfn main() {\n let x = 5;\n let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable\n}\n```\n\n### A Use Case for Interior Mutability: Mock Objects\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n```\na problematic usage\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n struct MockMessenger {\n sent_messages: Vec,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: vec![],\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.len(), 1);\n }\n}\n\n```\nWe can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition\n\nThis is a situation in which interior mutability can help! \n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::cell::RefCell;\n\n struct MockMessenger {\n sent_messages: RefCell>,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: RefCell::new(vec![]),\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell> in self.sent_messages to get a mutable reference to the value inside the RefCell>\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n // --snip--\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); // call borrow on the RefCell> to get an immutable reference to the vector.\n }\n}\n```\n\n### Keeping Track of Borrows at Runtime with RefCell\nWhen creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. **The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut**. Both types implement Deref, so we can treat them like regular references.\n\nThe RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime. \n```rust\nimpl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n let mut one_borrow = self.sent_messages.borrow_mut();\n let mut two_borrow = self.sent_messages.borrow_mut();\n\n one_borrow.push(String::from(message));\n two_borrow.push(String::from(message));\n }\n }\n```\nWhen we run the tests for our library, the code in will compile without any errors, but the test will fail\nthread 'main' panicked at 'already borrowed\n\n### Having Multiple Owners of Mutable Data by Combining Rc and RefCell\nA common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!\n```rust\n#[derive(Debug)]\nenum List {\n Cons(Rc>, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\nfn main() {\n let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc> and store it in a variable named value so we can access it directly later.\n\n let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); // Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc so when we create lists b and c, they can both refer to a\n\n let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));\n let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));\n\n *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc to the inner RefCell value. The borrow_mut method returns a RefMut smart pointer, and we use the dereference operator on it and change the inner value.\n\n println!(\"a after = {:?}\", a);\n println!(\"b after = {:?}\", b);\n println!(\"c after = {:?}\", c);\n}\n```\n\n**The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;** \n\n## Reference Cycles Can Leak Memory\nRust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).\n\n### Creating a Reference Cycle\n```rust\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>), // The second element in the Cons variant is now RefCell>, meaning that we want to modify which List value a Cons variant is pointing to. \n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n use crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>),\n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> {\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));\n\n println!(\"a initial rc count = {}\", Rc::strong_count(&a));\n println!(\"a next item = {:?}\", a.tail());\n\n let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a\n\n println!(\"a rc count after b creation = {}\", Rc::strong_count(&a));\n println!(\"b initial rc count = {}\", Rc::strong_count(&b));\n println!(\"b next item = {:?}\", b.tail());\n\n if let Some(link) = a.tail() {\n *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle\n }\n\n println!(\"b rc count after changing a = {}\", Rc::strong_count(&b));\n println!(\"a rc count after changing a = {}\", Rc::strong_count(&a));\n\n // Uncomment the next line to see that we have a cycle;\n // it will overflow the stack\n // println!(\"a next item = {:?}\", a.tail());\n} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc instance from 2 to 1. The memory that Rc has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc instance still refers to it. The memory allocated to the list will remain uncollected forever.\n```\n![cycle-ref](/images/rust/pointers/cycle.ref.png)\n\n### Preventing Reference Cycles: Turning an Rc into a Weak\nSo far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.\n\nStrong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.\n\nBecause the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped. \n\n\n### Creating a Tree Data Structure: a Node with Child Nodes\n```rust\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.\n children: RefCell>>,\n}\n\nfn main() {\n\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. \n\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. \n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak reference to branch from the Rc in branch.\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n\n \n}\n```\n\n### Visualizing Changes to strong_count and weak_count\n```rust\nuse std::cell::RefCell;\nuse std::rc::{Rc, Weak};\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>,\n children: RefCell>>,\n}\n\nfn main() {\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n\n {\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]),\n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch);\n\n println!(\n \"branch strong = {}, weak = {}\",\n Rc::strong_count(&branch),\n Rc::weak_count(&branch),\n );\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n }\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n}\n```","slug":"rust/rust-06-smart-pointer","published":1,"updated":"2023-05-03T07:31:43.613Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wps000j6hsj1nk1429h","content":"

Overview

The most common kind of pointer in Rust is a reference (borrow but not own)
Smart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.
references are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.

\n

smart pointers example

    \n
  • String
    • \n
  • Vec
    Both these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).
    • \n
\n

Deref & Drop

Smart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the Deref and Drop traits.

\n
    \n
  • The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers.
    • \n
  • The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.
    • \n
\n

the most common smart pointers in the standard library:

    \n
  • Box<T> for allocating values on the heap
    • \n
  • Rc<T>, a reference counting type that enables multiple ownership
    • \n
  • Ref<T> and RefMut<T>, accessed through RefCell<T>, a type that enforces the borrowing rules at runtime instead of compile time
    • \n
\n

Box

when to use

    \n
  • When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)
    • \n
  • When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so
    • \n
  • When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type
    • \n
\n

enabling recursive types with Box

At compile time, Rust needs to know how much space a type takes up
One type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address

\n
1
2
3
4
5
6
7
8
9
enum List {
    Cons(i32, List),
    Nil,
}

use crate::List::{Cons, Nil};
fn main() {
    let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size
}
\n\n

use Box

\n
1
2
3
4
5
6
7
8
9
10
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));
}
\n\n

Treating Smart Pointers Like Regular References with the Deref Trait

Implementing the Deref trait allows you to customize the behavior of the dereference operator, *
By implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.

\n

Defining Our Own Smart Pointer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
use std::ops::Deref;

struct MyBox<T>(T);  // The MyBox type is a tuple struct with one element of type T

impl<T> MyBox<T> {
    fn new(x: T) -> MyBox<T> {
        MyBox(x)
    }
}

impl<T> Deref for MyBox<T> {
    type Target = T;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

fn main() {
    let x = 5;
    let y = MyBox::new(x);

    assert_eq!(5, x);
    assert_eq!(5, *y);
}
\n
    \n
  • We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator.
    • \n
  • behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method
    • \n
  • The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.
    • \n
\n

Implicit Deref Coercions with Functions and Methods

Deref coercion is a convenience that Rust performs on arguments to functions and methods.
Deref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str
A sequence of calls to the deref method converts the type we provided into the type the parameter needs.

\n
1
2
3
4
5
6
7
8
9
10
fn hello(name: &str) {
    println!("Hello, {}!", name);
}

fn main() {
    let m = MyBox::new(String::from("Rust"));
    hello("rust");  // ok
    hello(&m);      // also ok
    hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox<String> into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello
}
\n

Here we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.

\n

How Deref Coercion Interacts with Mutability

Similar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.

\n

Rust does deref coercion when it finds types and trait implementations in three cases:

\n
    \n
  • From &T to &U when T: Deref<Target=U>
    • \n
  • From &mut T to &mut U when T: DerefMut<Target=U>
    • \n
  • From &mut T to &U when T: Deref<Target=U>
    • \n
\n

Running Code on Cleanup with the Drop Trait

Drop, which lets you customize what happens when a value is about to go out of scope.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("my stuff"),
    };
    c.drop(); // not allowed
    let d = CustomSmartPointer {
        data: String::from("other stuff"),
    };
    println!("CustomSmartPointers created.");
}

\n

Dropping a Value Early with std::mem::drop

std::mem::drop is in prelude, can use directly

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("some data"),
    };
    println!("CustomSmartPointer created.");
    drop(c); // ok
    println!("CustomSmartPointer dropped before the end of main.");
} // c goes out of scope, will occur double drop

\n\n\n

Rc: the Reference Counted Smart Pointer

In the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners.
type keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.

\n

We use the Rc<T> type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.

\n

Rc is only for use in single-threaded scenarios

\n

using Rc to share data

\"rc\"
implement use box will not work, as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));
    let b = Cons(3, Box::new(a)); // value moved here
    let c = Cons(4, Box::new(a)); // value used here after move
}
\n

use Rc

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data
    let c = Cons(4, Rc::clone(&a));
}
\n\n\n

We could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.

\n

Cloning an Rc Increases the Reference Count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    println!("count after creating a = {}", Rc::strong_count(&a));  // 1
    let b = Cons(3, Rc::clone(&a));
    println!("count after creating b = {}", Rc::strong_count(&a));  // 2
    {
        let c = Cons(4, Rc::clone(&a));
        println!("count after creating c = {}", Rc::strong_count(&a)); // 3
    }
    println!("count after c goes out of scope = {}", Rc::strong_count(&a));  // 2
}
\n

What we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.

\n

RefCell and interior mutability pattern

Interior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.
To mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.
We can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that.
The unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.

\n

Enforcing Borrowing Rules at Runtime with RefCell

Unlike Rc, the RefCell type represents single ownership over the data it holds.

\n

recall borrowing rules

\n
    \n
  • At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.
    • \n
  • References must always be valid.
    With references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime.
    Because RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.
    • \n
\n

Interior Mutability: A Mutable Borrow to an Immutable Value

1
2
3
4
fn main() {
    let x = 5;
    let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable
}
\n\n

A Use Case for Interior Mutability: Mock Objects

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

\n

a problematic usage

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    struct MockMessenger {
        sent_messages: Vec<String>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: vec![],
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.len(), 1);
    }
}

\n

We can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition

\n

This is a situation in which interior mutability can help!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::RefCell;

    struct MockMessenger {
        sent_messages: RefCell<Vec<String>>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: RefCell::new(vec![]),
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell<Vec<String>> in self.sent_messages to get a mutable reference to the value inside the RefCell<Vec<String>>
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        // --snip--
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); //  call borrow on the RefCell<Vec<String>> to get an immutable reference to the vector.
    }
}
\n\n

Keeping Track of Borrows at Runtime with RefCell

When creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut. Both types implement Deref, so we can treat them like regular references.

\n

The RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime.

\n
1
2
3
4
5
6
7
8
9
impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            let mut one_borrow = self.sent_messages.borrow_mut();
            let mut two_borrow = self.sent_messages.borrow_mut();

            one_borrow.push(String::from(message));
            two_borrow.push(String::from(message));
        }
    }
\n

When we run the tests for our library, the code in will compile without any errors, but the test will fail
thread ‘main’ panicked at ‘already borrowed

\n

Having Multiple Owners of Mutable Data by Combining Rc and RefCell

A common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#[derive(Debug)]
enum List {
    Cons(Rc<RefCell<i32>>, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

fn main() {
    let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc<RefCell<i32>> and store it in a variable named value so we can access it directly later.

    let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); //  Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc<T> so when we create lists b and c, they can both refer to a

    let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));
    let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));

    *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc<T> to the inner RefCell<T> value. The borrow_mut method returns a RefMut<T> smart pointer, and we use the dereference operator on it and change the inner value.

    println!("a after = {:?}", a);
    println!("b after = {:?}", b);
    println!("c after = {:?}", c);
}
\n\n

The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;

\n

Reference Cycles Can Leak Memory

Rust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).

\n

Creating a Reference Cycle

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>), // The second element in the Cons variant is now RefCell<Rc<List>>, meaning that we want to modify which List value a Cons variant is pointing to. 
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>),
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> {
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));

    println!("a initial rc count = {}", Rc::strong_count(&a));
    println!("a next item = {:?}", a.tail());

    let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a

    println!("a rc count after b creation = {}", Rc::strong_count(&a));
    println!("b initial rc count = {}", Rc::strong_count(&b));
    println!("b next item = {:?}", b.tail());

    if let Some(link) = a.tail() {
        *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle
    }

    println!("b rc count after changing a = {}", Rc::strong_count(&b));
    println!("a rc count after changing a = {}", Rc::strong_count(&a));

    // Uncomment the next line to see that we have a cycle;
    // it will overflow the stack
    // println!("a next item = {:?}", a.tail());
} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc<List> instance from 2 to 1. The memory that Rc<List> has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc<List> instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc<List> instance still refers to it. The memory allocated to the list will remain uncollected forever.
\n

\"cycle-ref\"

\n

Preventing Reference Cycles: Turning an Rc into a Weak

So far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.

\n

Strong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.

\n

Because the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option<Rc>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped.

\n

Creating a Tree Data Structure: a Node with Child Nodes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc<T>, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {

    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. 

    let branch = Rc::new(Node {
        value: 5,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc<Node> in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. 
    });

    *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak<Node> reference to branch from the Rc<Node> in branch.

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());

   
}
\n\n

Visualizing Changes to strong_count and weak_count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
use std::cell::RefCell;
use std::rc::{Rc, Weak};

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>,
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {
    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );

    {
        let branch = Rc::new(Node {
            value: 5,
            parent: RefCell::new(Weak::new()),
            children: RefCell::new(vec![Rc::clone(&leaf)]),
        });

        *leaf.parent.borrow_mut() = Rc::downgrade(&branch);

        println!(
            "branch strong = {}, weak = {}",
            Rc::strong_count(&branch),
            Rc::weak_count(&branch),
        );

        println!(
            "leaf strong = {}, weak = {}",
            Rc::strong_count(&leaf),
            Rc::weak_count(&leaf),
        );
    }

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());
    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );
}
","site":{"data":{}},"excerpt":"","more":"

Overview

The most common kind of pointer in Rust is a reference (borrow but not own)
Smart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.
references are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.

\n

smart pointers example

    \n
  • String
    • \n
  • Vec
    Both these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).
    • \n
\n

Deref & Drop

Smart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the Deref and Drop traits.

\n
    \n
  • The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers.
    • \n
  • The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.
    • \n
\n

the most common smart pointers in the standard library:

    \n
  • Box<T> for allocating values on the heap
    • \n
  • Rc<T>, a reference counting type that enables multiple ownership
    • \n
  • Ref<T> and RefMut<T>, accessed through RefCell<T>, a type that enforces the borrowing rules at runtime instead of compile time
    • \n
\n

Box

when to use

    \n
  • When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)
    • \n
  • When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so
    • \n
  • When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type
    • \n
\n

enabling recursive types with Box

At compile time, Rust needs to know how much space a type takes up
One type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address

\n
1
2
3
4
5
6
7
8
9
enum List {
    Cons(i32, List),
    Nil,
}

use crate::List::{Cons, Nil};
fn main() {
    let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size
}
\n\n

use Box

\n
1
2
3
4
5
6
7
8
9
10
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));
}
\n\n

Treating Smart Pointers Like Regular References with the Deref Trait

Implementing the Deref trait allows you to customize the behavior of the dereference operator, *
By implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.

\n

Defining Our Own Smart Pointer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
use std::ops::Deref;

struct MyBox<T>(T);  // The MyBox type is a tuple struct with one element of type T

impl<T> MyBox<T> {
    fn new(x: T) -> MyBox<T> {
        MyBox(x)
    }
}

impl<T> Deref for MyBox<T> {
    type Target = T;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

fn main() {
    let x = 5;
    let y = MyBox::new(x);

    assert_eq!(5, x);
    assert_eq!(5, *y);
}
\n
    \n
  • We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator.
    • \n
  • behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method
    • \n
  • The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.
    • \n
\n

Implicit Deref Coercions with Functions and Methods

Deref coercion is a convenience that Rust performs on arguments to functions and methods.
Deref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str
A sequence of calls to the deref method converts the type we provided into the type the parameter needs.

\n
1
2
3
4
5
6
7
8
9
10
fn hello(name: &str) {
    println!("Hello, {}!", name);
}

fn main() {
    let m = MyBox::new(String::from("Rust"));
    hello("rust");  // ok
    hello(&m);      // also ok
    hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox<String> into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello
}
\n

Here we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.

\n

How Deref Coercion Interacts with Mutability

Similar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.

\n

Rust does deref coercion when it finds types and trait implementations in three cases:

\n
    \n
  • From &T to &U when T: Deref<Target=U>
    • \n
  • From &mut T to &mut U when T: DerefMut<Target=U>
    • \n
  • From &mut T to &U when T: Deref<Target=U>
    • \n
\n

Running Code on Cleanup with the Drop Trait

Drop, which lets you customize what happens when a value is about to go out of scope.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("my stuff"),
    };
    c.drop(); // not allowed
    let d = CustomSmartPointer {
        data: String::from("other stuff"),
    };
    println!("CustomSmartPointers created.");
}

\n

Dropping a Value Early with std::mem::drop

std::mem::drop is in prelude, can use directly

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("some data"),
    };
    println!("CustomSmartPointer created.");
    drop(c); // ok
    println!("CustomSmartPointer dropped before the end of main.");
} // c goes out of scope, will occur double drop

\n\n\n

Rc: the Reference Counted Smart Pointer

In the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners.
type keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.

\n

We use the Rc<T> type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.

\n

Rc is only for use in single-threaded scenarios

\n

using Rc to share data

\"rc\"
implement use box will not work, as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));
    let b = Cons(3, Box::new(a)); // value moved here
    let c = Cons(4, Box::new(a)); // value used here after move
}
\n

use Rc

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data
    let c = Cons(4, Rc::clone(&a));
}
\n\n\n

We could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.

\n

Cloning an Rc Increases the Reference Count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    println!("count after creating a = {}", Rc::strong_count(&a));  // 1
    let b = Cons(3, Rc::clone(&a));
    println!("count after creating b = {}", Rc::strong_count(&a));  // 2
    {
        let c = Cons(4, Rc::clone(&a));
        println!("count after creating c = {}", Rc::strong_count(&a)); // 3
    }
    println!("count after c goes out of scope = {}", Rc::strong_count(&a));  // 2
}
\n

What we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.

\n

RefCell and interior mutability pattern

Interior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.
To mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.
We can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that.
The unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.

\n

Enforcing Borrowing Rules at Runtime with RefCell

Unlike Rc, the RefCell type represents single ownership over the data it holds.

\n

recall borrowing rules

\n
    \n
  • At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.
    • \n
  • References must always be valid.
    With references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime.
    Because RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.
    • \n
\n

Interior Mutability: A Mutable Borrow to an Immutable Value

1
2
3
4
fn main() {
    let x = 5;
    let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable
}
\n\n

A Use Case for Interior Mutability: Mock Objects

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

\n

a problematic usage

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    struct MockMessenger {
        sent_messages: Vec<String>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: vec![],
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.len(), 1);
    }
}

\n

We can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition

\n

This is a situation in which interior mutability can help!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::RefCell;

    struct MockMessenger {
        sent_messages: RefCell<Vec<String>>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: RefCell::new(vec![]),
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell<Vec<String>> in self.sent_messages to get a mutable reference to the value inside the RefCell<Vec<String>>
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        // --snip--
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); //  call borrow on the RefCell<Vec<String>> to get an immutable reference to the vector.
    }
}
\n\n

Keeping Track of Borrows at Runtime with RefCell

When creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut. Both types implement Deref, so we can treat them like regular references.

\n

The RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime.

\n
1
2
3
4
5
6
7
8
9
impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            let mut one_borrow = self.sent_messages.borrow_mut();
            let mut two_borrow = self.sent_messages.borrow_mut();

            one_borrow.push(String::from(message));
            two_borrow.push(String::from(message));
        }
    }
\n

When we run the tests for our library, the code in will compile without any errors, but the test will fail
thread ‘main’ panicked at ‘already borrowed

\n

Having Multiple Owners of Mutable Data by Combining Rc and RefCell

A common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#[derive(Debug)]
enum List {
    Cons(Rc<RefCell<i32>>, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

fn main() {
    let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc<RefCell<i32>> and store it in a variable named value so we can access it directly later.

    let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); //  Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc<T> so when we create lists b and c, they can both refer to a

    let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));
    let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));

    *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc<T> to the inner RefCell<T> value. The borrow_mut method returns a RefMut<T> smart pointer, and we use the dereference operator on it and change the inner value.

    println!("a after = {:?}", a);
    println!("b after = {:?}", b);
    println!("c after = {:?}", c);
}
\n\n

The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;

\n

Reference Cycles Can Leak Memory

Rust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).

\n

Creating a Reference Cycle

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>), // The second element in the Cons variant is now RefCell<Rc<List>>, meaning that we want to modify which List value a Cons variant is pointing to. 
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>),
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> {
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));

    println!("a initial rc count = {}", Rc::strong_count(&a));
    println!("a next item = {:?}", a.tail());

    let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a

    println!("a rc count after b creation = {}", Rc::strong_count(&a));
    println!("b initial rc count = {}", Rc::strong_count(&b));
    println!("b next item = {:?}", b.tail());

    if let Some(link) = a.tail() {
        *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle
    }

    println!("b rc count after changing a = {}", Rc::strong_count(&b));
    println!("a rc count after changing a = {}", Rc::strong_count(&a));

    // Uncomment the next line to see that we have a cycle;
    // it will overflow the stack
    // println!("a next item = {:?}", a.tail());
} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc<List> instance from 2 to 1. The memory that Rc<List> has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc<List> instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc<List> instance still refers to it. The memory allocated to the list will remain uncollected forever.
\n

\"cycle-ref\"

\n

Preventing Reference Cycles: Turning an Rc into a Weak

So far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.

\n

Strong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.

\n

Because the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option<Rc>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped.

\n

Creating a Tree Data Structure: a Node with Child Nodes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc<T>, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {

    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. 

    let branch = Rc::new(Node {
        value: 5,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc<Node> in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. 
    });

    *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak<Node> reference to branch from the Rc<Node> in branch.

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());

   
}
\n\n

Visualizing Changes to strong_count and weak_count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
use std::cell::RefCell;
use std::rc::{Rc, Weak};

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>,
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {
    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );

    {
        let branch = Rc::new(Node {
            value: 5,
            parent: RefCell::new(Weak::new()),
            children: RefCell::new(vec![Rc::clone(&leaf)]),
        });

        *leaf.parent.borrow_mut() = Rc::downgrade(&branch);

        println!(
            "branch strong = {}, weak = {}",
            Rc::strong_count(&branch),
            Rc::weak_count(&branch),
        );

        println!(
            "leaf strong = {}, weak = {}",
            Rc::strong_count(&leaf),
            Rc::weak_count(&leaf),
        );
    }

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());
    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );
}
"},{"title":"rust reflect and macro","date":"2022-12-28T02:24:21.000Z","_content":"\n## reflect\n### trait object\nRust provides dynamic dispatch through a feature called `trait objects`. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at **runtime**. more details can be found on [references 1]\n\n### any\nThis module (std::any) contains the `Any` trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the `Provider` trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.\nAny itself can be used to get a TypeId\n\n```rust\nuse std::fmt::Debug;\nuse std::any::Any;\n\nfn log(value: &Any) {\n let value_any = value as &dyn Any; // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.\n\n match value_any.downcast_ref::() {\n Some(val_str) -> {\n // do with string\n },\n None => {\n // \n }\n }\n}\n```\n\n### porpular crates using Any\n- [oso](https://docs.osohq.com/)\nThe Oso Library is a batteries-included framework for building authorization in your application\n- [bevy](https://bevyengine.org/)\nA refreshingly simple data-driven game engine built in Rust\n\n## macro\n### rust compile process\n![rust compile process](/images/rust/macros/16.compile_process.png)\n\n### front end: rustc\n1. lexical analysis: Code Text -> TokenStream\n2. syntax analysis: TokenStream -> AST (abstract syntax tree)\n3. semantic analyzer: \n AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g `for` changed to `loop`) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)\n\n### back end: LLVM\nLLVM IR -> machine code\n\n\n### macros in compiling\n- declarative macros: TokenStream - expand -> TokenStream\n- procedule macros: self defined AST with the help or third party crate such as syn, quote\n\n## declarative macro: macro_rules!\nDeclarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed\n```rust\n#![allow(unused)]\nfn main() {\n match target {\n match_pattern_1 => expr_1,\n match_pattern_2 => {\n statement1;\n statement2;\n expr_2\n },\n _ => expr_3\n }\n}\n```\n\n### example 1, simplified `vec!`\nbelow example use macro_rules to implement a simplified version of vec!\n```rust\n#[macro_export]\nmacro_rules! vec {\n ( $( $x:expr ),* ) => {\n {\n let mut temp_vec = Vec::new();\n $(\n temp_vec.push($x);\n )*\n temp_vec\n }\n };\n}\n\n#![allow(unused)]\nfn main() {\n let v: Vec = vec![1, 2, 3];\n}\n```\n\n### example 2, unless\n```rust\nmacro_rules! unless {\n ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);\n}\n\nfn cmp(a:i32, b:i32) {\n unless!{\n (a>b) => {println!(\"{} < {}\", a,b);}\n }\n}\nfn main() {\n cmp(1,2); /// print \"1<2\" as the condition is true !(a>b)\n cmp(3,2); /// print nothing\n}\n```\n### example 3, HashMap\n```rust\nmacro_rules! hashmap {\n // match for \"a\" => 1, \"b\" => 2,\n ( $($key:expr => $value:expr,)* ) =>\n { hashmap!($($key => $value),*) }; // recuisive\n // match for \"a\" => 1, \"b\" => 2\n ( $($key:expr => $value:expr),* ) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\n\nmacro_rules! hashmap_equivalent {\n ( $($key:expr => $value:expr),* $(,)*) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\nfn main() {\n let map = hashmap!{\n \"a\" => 1,\n \"b\" => 2, // with or without ,\n };\n let map_2 = hashmap_equivalent!{\n \"a\" => 1, \n \"b\" => 2, // with or without ,\n };\n}\n```\n\n### metavariables\n- item: an Item\n- stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)\n- expr: an Expression\n- ty: a Type\n- ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER\n- path: a TypePath style path\n- tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})\n- meta: an Attr, the contents of an attribute\n- lifetime: a LIFETIME_TOKEN\n- vis: a possibly empty Visibility qualifier\n- literal: matches LiteralExpression\ndetails to be found [here](https://doc.rust-lang.org/reference/macros-by-example.html)\n\n## procedures macro\n\n## references\n1. [rust trait object](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/trait-objects.html)","source":"_posts/rust/rust-07-macro.md","raw":"---\ntitle: rust reflect and macro\ndate: 2022-12-28 10:24:21\ntags: [rust]\n---\n\n## reflect\n### trait object\nRust provides dynamic dispatch through a feature called `trait objects`. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at **runtime**. more details can be found on [references 1]\n\n### any\nThis module (std::any) contains the `Any` trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the `Provider` trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.\nAny itself can be used to get a TypeId\n\n```rust\nuse std::fmt::Debug;\nuse std::any::Any;\n\nfn log(value: &Any) {\n let value_any = value as &dyn Any; // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.\n\n match value_any.downcast_ref::() {\n Some(val_str) -> {\n // do with string\n },\n None => {\n // \n }\n }\n}\n```\n\n### porpular crates using Any\n- [oso](https://docs.osohq.com/)\nThe Oso Library is a batteries-included framework for building authorization in your application\n- [bevy](https://bevyengine.org/)\nA refreshingly simple data-driven game engine built in Rust\n\n## macro\n### rust compile process\n![rust compile process](/images/rust/macros/16.compile_process.png)\n\n### front end: rustc\n1. lexical analysis: Code Text -> TokenStream\n2. syntax analysis: TokenStream -> AST (abstract syntax tree)\n3. semantic analyzer: \n AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g `for` changed to `loop`) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)\n\n### back end: LLVM\nLLVM IR -> machine code\n\n\n### macros in compiling\n- declarative macros: TokenStream - expand -> TokenStream\n- procedule macros: self defined AST with the help or third party crate such as syn, quote\n\n## declarative macro: macro_rules!\nDeclarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed\n```rust\n#![allow(unused)]\nfn main() {\n match target {\n match_pattern_1 => expr_1,\n match_pattern_2 => {\n statement1;\n statement2;\n expr_2\n },\n _ => expr_3\n }\n}\n```\n\n### example 1, simplified `vec!`\nbelow example use macro_rules to implement a simplified version of vec!\n```rust\n#[macro_export]\nmacro_rules! vec {\n ( $( $x:expr ),* ) => {\n {\n let mut temp_vec = Vec::new();\n $(\n temp_vec.push($x);\n )*\n temp_vec\n }\n };\n}\n\n#![allow(unused)]\nfn main() {\n let v: Vec = vec![1, 2, 3];\n}\n```\n\n### example 2, unless\n```rust\nmacro_rules! unless {\n ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);\n}\n\nfn cmp(a:i32, b:i32) {\n unless!{\n (a>b) => {println!(\"{} < {}\", a,b);}\n }\n}\nfn main() {\n cmp(1,2); /// print \"1<2\" as the condition is true !(a>b)\n cmp(3,2); /// print nothing\n}\n```\n### example 3, HashMap\n```rust\nmacro_rules! hashmap {\n // match for \"a\" => 1, \"b\" => 2,\n ( $($key:expr => $value:expr,)* ) =>\n { hashmap!($($key => $value),*) }; // recuisive\n // match for \"a\" => 1, \"b\" => 2\n ( $($key:expr => $value:expr),* ) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\n\nmacro_rules! hashmap_equivalent {\n ( $($key:expr => $value:expr),* $(,)*) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\nfn main() {\n let map = hashmap!{\n \"a\" => 1,\n \"b\" => 2, // with or without ,\n };\n let map_2 = hashmap_equivalent!{\n \"a\" => 1, \n \"b\" => 2, // with or without ,\n };\n}\n```\n\n### metavariables\n- item: an Item\n- stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)\n- expr: an Expression\n- ty: a Type\n- ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER\n- path: a TypePath style path\n- tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})\n- meta: an Attr, the contents of an attribute\n- lifetime: a LIFETIME_TOKEN\n- vis: a possibly empty Visibility qualifier\n- literal: matches LiteralExpression\ndetails to be found [here](https://doc.rust-lang.org/reference/macros-by-example.html)\n\n## procedures macro\n\n## references\n1. [rust trait object](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/trait-objects.html)","slug":"rust/rust-07-macro","published":1,"updated":"2023-05-01T14:18:30.350Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wps000l6hsja9odhbdf","content":"

reflect

trait object

Rust provides dynamic dispatch through a feature called trait objects. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at runtime. more details can be found on [references 1]

\n

any

This module (std::any) contains the Any trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the Provider trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.
Any itself can be used to get a TypeId

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
use std::fmt::Debug;
use std::any::Any;

fn log<T: Any + Debug>(value: &Any) {
    let value_any = value as &dyn Any;  // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.

    match value_any.downcast_ref::<String>() {
        Some(val_str) -> {
            // do with string
        },
        None => {
            // 
        }
    }
}
\n\n

porpular crates using Any

    \n
  • oso
    The Oso Library is a batteries-included framework for building authorization in your application
    • \n
  • bevy
    A refreshingly simple data-driven game engine built in Rust
    • \n
\n

macro

rust compile process

\"rust

\n

front end: rustc

    \n
  1. lexical analysis: Code Text -> TokenStream
    2. \n
  2. syntax analysis: TokenStream -> AST (abstract syntax tree)
    3. \n
  3. semantic analyzer:
    AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g for changed to loop) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)
    4. \n
\n

back end: LLVM

LLVM IR -> machine code

\n

macros in compiling

    \n
  • declarative macros: TokenStream - expand -> TokenStream
    • \n
  • procedule macros: self defined AST with the help or third party crate such as syn, quote
    • \n
\n

declarative macro: macro_rules!

Declarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed

\n
1
2
3
4
5
6
7
8
9
10
11
12
#![allow(unused)]
fn main() {
    match target {
        match_pattern_1 => expr_1,
        match_pattern_2 => {
            statement1;
            statement2;
            expr_2
        },
        _ => expr_3
    }
}
\n\n

example 1, simplified vec!

below example use macro_rules to implement a simplified version of vec!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#[macro_export]
macro_rules! vec {
    ( $( $x:expr ),* ) => {
        {
            let mut temp_vec = Vec::new();
            $(
                temp_vec.push($x);
            )*
            temp_vec
        }
    };
}

#![allow(unused)]
fn main() {
    let v: Vec<u32> = vec![1, 2, 3];
}
\n\n

example 2, unless

1
2
3
4
5
6
7
8
9
10
11
12
13
macro_rules! unless {
    ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);
}

fn cmp(a:i32, b:i32) {
    unless!{
        (a>b) => {println!("{} < {}", a,b);}
    }
}
fn main() {
    cmp(1,2);  /// print "1<2" as the condition is true !(a>b)
    cmp(3,2);  /// print nothing
}
\n

example 3, HashMap

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
macro_rules! hashmap {
    // match for "a" => 1, "b" => 2,
    ( $($key:expr => $value:expr,)* ) =>
        { hashmap!($($key => $value),*) }; // recuisive
    // match for "a" => 1, "b" => 2
    ( $($key:expr => $value:expr),* ) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}

macro_rules! hashmap_equivalent {
    ( $($key:expr => $value:expr),* $(,)*) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}
fn main() {
    let map = hashmap!{
        "a" => 1,
        "b" => 2,  // with or without ,
    };
    let map_2 = hashmap_equivalent!{
        "a" => 1, 
        "b" => 2,  // with or without ,
    };
}
\n\n

metavariables

    \n
  • item: an Item
    • \n
  • stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)
    • \n
  • expr: an Expression
    • \n
  • ty: a Type
    • \n
  • ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER
    • \n
  • path: a TypePath style path
    • \n
  • tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})
    • \n
  • meta: an Attr, the contents of an attribute
    • \n
  • lifetime: a LIFETIME_TOKEN
    • \n
  • vis: a possibly empty Visibility qualifier
    • \n
  • literal: matches LiteralExpression
    details to be found here
    • \n
\n

procedures macro

references

    \n
  1. rust trait object
    2. \n
\n","site":{"data":{}},"excerpt":"","more":"

reflect

trait object

Rust provides dynamic dispatch through a feature called trait objects. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at runtime. more details can be found on [references 1]

\n

any

This module (std::any) contains the Any trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the Provider trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.
Any itself can be used to get a TypeId

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
use std::fmt::Debug;
use std::any::Any;

fn log<T: Any + Debug>(value: &Any) {
    let value_any = value as &dyn Any;  // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.

    match value_any.downcast_ref::<String>() {
        Some(val_str) -> {
            // do with string
        },
        None => {
            // 
        }
    }
}
\n\n

porpular crates using Any

    \n
  • oso
    The Oso Library is a batteries-included framework for building authorization in your application
    • \n
  • bevy
    A refreshingly simple data-driven game engine built in Rust
    • \n
\n

macro

rust compile process

\"rust

\n

front end: rustc

    \n
  1. lexical analysis: Code Text -> TokenStream
    2. \n
  2. syntax analysis: TokenStream -> AST (abstract syntax tree)
    3. \n
  3. semantic analyzer:
    AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g for changed to loop) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)
    4. \n
\n

back end: LLVM

LLVM IR -> machine code

\n

macros in compiling

    \n
  • declarative macros: TokenStream - expand -> TokenStream
    • \n
  • procedule macros: self defined AST with the help or third party crate such as syn, quote
    • \n
\n

declarative macro: macro_rules!

Declarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed

\n
1
2
3
4
5
6
7
8
9
10
11
12
#![allow(unused)]
fn main() {
    match target {
        match_pattern_1 => expr_1,
        match_pattern_2 => {
            statement1;
            statement2;
            expr_2
        },
        _ => expr_3
    }
}
\n\n

example 1, simplified vec!

below example use macro_rules to implement a simplified version of vec!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#[macro_export]
macro_rules! vec {
    ( $( $x:expr ),* ) => {
        {
            let mut temp_vec = Vec::new();
            $(
                temp_vec.push($x);
            )*
            temp_vec
        }
    };
}

#![allow(unused)]
fn main() {
    let v: Vec<u32> = vec![1, 2, 3];
}
\n\n

example 2, unless

1
2
3
4
5
6
7
8
9
10
11
12
13
macro_rules! unless {
    ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);
}

fn cmp(a:i32, b:i32) {
    unless!{
        (a>b) => {println!("{} < {}", a,b);}
    }
}
fn main() {
    cmp(1,2);  /// print "1<2" as the condition is true !(a>b)
    cmp(3,2);  /// print nothing
}
\n

example 3, HashMap

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
macro_rules! hashmap {
    // match for "a" => 1, "b" => 2,
    ( $($key:expr => $value:expr,)* ) =>
        { hashmap!($($key => $value),*) }; // recuisive
    // match for "a" => 1, "b" => 2
    ( $($key:expr => $value:expr),* ) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}

macro_rules! hashmap_equivalent {
    ( $($key:expr => $value:expr),* $(,)*) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}
fn main() {
    let map = hashmap!{
        "a" => 1,
        "b" => 2,  // with or without ,
    };
    let map_2 = hashmap_equivalent!{
        "a" => 1, 
        "b" => 2,  // with or without ,
    };
}
\n\n

metavariables

    \n
  • item: an Item
    • \n
  • stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)
    • \n
  • expr: an Expression
    • \n
  • ty: a Type
    • \n
  • ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER
    • \n
  • path: a TypePath style path
    • \n
  • tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})
    • \n
  • meta: an Attr, the contents of an attribute
    • \n
  • lifetime: a LIFETIME_TOKEN
    • \n
  • vis: a possibly empty Visibility qualifier
    • \n
  • literal: matches LiteralExpression
    details to be found here
    • \n
\n

procedures macro

references

    \n
  1. rust trait object
    2. \n
\n"},{"title":"rust functional programming","date":"2023-04-11T14:04:38.000Z","_content":"\n## iterator\n- `iter()` Returns an iterator over the **slice**\n- `into_iter()` Creates a consuming iterator, that is, one that moves each value out of the vector\n- `iter_mut()` Returns an iterator that allows modifying each value.\n\n## flat_map\n```rust\n/// Creates an iterator that works like map, but flattens nested structure.\n///\n/// The [`map`] adapter is very useful, but only when the closure\n/// argument produces values. If it produces an iterator instead, there's\n/// an extra layer of indirection. `flat_map()` will remove this extra layer\n/// on its own.\n```\n### Examples\n```rust\nlet words = [\"alpha\", \"beta\", \"gamma\"];\n// chars() returns an iterator\nlet merged: String = words.iter()\n .flat_map(|s| s.chars())\n .collect();\nassert_eq!(merged, \"alphabetagamma\");\n```\n","source":"_posts/rust/rust-09-functional.md","raw":"---\ntitle: rust functional programming\ndate: 2023-04-11 22:04:38\ntags: [rust]\n---\n\n## iterator\n- `iter()` Returns an iterator over the **slice**\n- `into_iter()` Creates a consuming iterator, that is, one that moves each value out of the vector\n- `iter_mut()` Returns an iterator that allows modifying each value.\n\n## flat_map\n```rust\n/// Creates an iterator that works like map, but flattens nested structure.\n///\n/// The [`map`] adapter is very useful, but only when the closure\n/// argument produces values. If it produces an iterator instead, there's\n/// an extra layer of indirection. `flat_map()` will remove this extra layer\n/// on its own.\n```\n### Examples\n```rust\nlet words = [\"alpha\", \"beta\", \"gamma\"];\n// chars() returns an iterator\nlet merged: String = words.iter()\n .flat_map(|s| s.chars())\n .collect();\nassert_eq!(merged, \"alphabetagamma\");\n```\n","slug":"rust/rust-09-functional","published":1,"updated":"2023-06-29T01:53:41.688Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wps000n6hsj472tg11o","content":"

iterator

    \n
  • iter() Returns an iterator over the slice
    • \n
  • into_iter() Creates a consuming iterator, that is, one that moves each value out of the vector
    • \n
  • iter_mut() Returns an iterator that allows modifying each value.
    • \n
\n

flat_map

1
2
3
4
5
6
/// Creates an iterator that works like map, but flattens nested structure.
///
/// The [`map`] adapter is very useful, but only when the closure
/// argument produces values. If it produces an iterator instead, there's
/// an extra layer of indirection. `flat_map()` will remove this extra layer
/// on its own.
\n

Examples

1
2
3
4
5
6
let words = ["alpha", "beta", "gamma"];
// chars() returns an iterator
let merged: String = words.iter()
                          .flat_map(|s| s.chars())
                          .collect();
assert_eq!(merged, "alphabetagamma");
\n","site":{"data":{}},"excerpt":"","more":"

iterator

    \n
  • iter() Returns an iterator over the slice
    • \n
  • into_iter() Creates a consuming iterator, that is, one that moves each value out of the vector
    • \n
  • iter_mut() Returns an iterator that allows modifying each value.
    • \n
\n

flat_map

1
2
3
4
5
6
/// Creates an iterator that works like map, but flattens nested structure.
///
/// The [`map`] adapter is very useful, but only when the closure
/// argument produces values. If it produces an iterator instead, there's
/// an extra layer of indirection. `flat_map()` will remove this extra layer
/// on its own.
\n

Examples

1
2
3
4
5
6
let words = ["alpha", "beta", "gamma"];
// chars() returns an iterator
let merged: String = words.iter()
                          .flat_map(|s| s.chars())
                          .collect();
assert_eq!(merged, "alphabetagamma");
\n"},{"title":"rust project management","date":"2022-11-06T07:33:55.000Z","_content":"\n## basic concepts\n- **Packages**: A Cargo feature that lets you build, test, and share crates\n- **Crates**: A tree of modules that produces a library or executable\n- **Modules and use**: Let you control the organization, scope, and privacy of paths\n- **Paths**: A way of naming an item, such as a struct, function, or module\n\n### package\n1. one package can only has one library Crate. default is src/lib.rs\n2. one package can have multiple binary Crates (under src/bin). default src/main.rs\n3. one package has at least one crate，no matter lib or bin。\n4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)\n```\ncargo new my-project --lib ## create a library project\n```\n### crate\n- binary or library\n- The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate\n\n### module\n- keyword `mod` \n- module could be nested。\n- module includes(struct、enum、const、trait、func, etc)\n\n### path\n- absolute path: use crate name or `crate`\n- relative path:，use self, super etc\n```rust\nfn serve_order() {}\nmod back_of_house {\n fn fix_incorrect_order() {\n cook_order();\n super::serve_order();\n }\n fn cook_order() {}\n}\n```\n```rust\nmod front_of_house {\n pub mod hosting {\n pub fn add_to_waitlist() {}\n }\n}\npub fn eat_at_restaurant() {\n // Absolute path\n crate::front_of_house::hosting::add_to_waitlist();\n // Relative path\n front_of_house::hosting::add_to_waitlist();\n}\n```\n\n### Bringing Paths into Scope with the use Keyword\n```rust\nuse std::io;\nuse std::io::Write;\nuse std::io:: { self, Write} ;\nuse std::{cmp::Ordering, io};\nuse std::fmt::Result;\nuse std::io::Result as IoResult;\nuse std::collections::*;\n\npub use crate::front_of_house::hosting; // re-exporting names with pub use\n```\n### Separating Modules into Different Files\nFilename: src/lib.rs\n```rust\nmod front_of_house;\n\npub use crate::front_of_house::hosting;\n\npub fn eat_at_restaurant() {\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n}\n```\nFilename: src/front_of_house.rs\n```rust\npub mod hosting {\n pub fn add_to_waitlist() {}\n}\n```\n\n## profile\n```\ncargo build ## dev: [unoptimized + debuginfo]\ncargo build --release ## [optimized]\n```\n- config\n```\n[profile.dev]\nopt-level = 0\n\n[profile.release]\nopt-level = 3\n\n# profile for the wasm example (from project ethers-rs)\n[profile.release.package.ethers-wasm]\nopt-level = \"s\" # Tell `rustc` to optimize for small code size.\n```\n","source":"_posts/rust/rust-08-project-management.md","raw":"---\ntitle: rust project management\ndate: 2022-11-06 15:33:55\ntags: [rust]\n---\n\n## basic concepts\n- **Packages**: A Cargo feature that lets you build, test, and share crates\n- **Crates**: A tree of modules that produces a library or executable\n- **Modules and use**: Let you control the organization, scope, and privacy of paths\n- **Paths**: A way of naming an item, such as a struct, function, or module\n\n### package\n1. one package can only has one library Crate. default is src/lib.rs\n2. one package can have multiple binary Crates (under src/bin). default src/main.rs\n3. one package has at least one crate，no matter lib or bin。\n4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)\n```\ncargo new my-project --lib ## create a library project\n```\n### crate\n- binary or library\n- The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate\n\n### module\n- keyword `mod` \n- module could be nested。\n- module includes(struct、enum、const、trait、func, etc)\n\n### path\n- absolute path: use crate name or `crate`\n- relative path:，use self, super etc\n```rust\nfn serve_order() {}\nmod back_of_house {\n fn fix_incorrect_order() {\n cook_order();\n super::serve_order();\n }\n fn cook_order() {}\n}\n```\n```rust\nmod front_of_house {\n pub mod hosting {\n pub fn add_to_waitlist() {}\n }\n}\npub fn eat_at_restaurant() {\n // Absolute path\n crate::front_of_house::hosting::add_to_waitlist();\n // Relative path\n front_of_house::hosting::add_to_waitlist();\n}\n```\n\n### Bringing Paths into Scope with the use Keyword\n```rust\nuse std::io;\nuse std::io::Write;\nuse std::io:: { self, Write} ;\nuse std::{cmp::Ordering, io};\nuse std::fmt::Result;\nuse std::io::Result as IoResult;\nuse std::collections::*;\n\npub use crate::front_of_house::hosting; // re-exporting names with pub use\n```\n### Separating Modules into Different Files\nFilename: src/lib.rs\n```rust\nmod front_of_house;\n\npub use crate::front_of_house::hosting;\n\npub fn eat_at_restaurant() {\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n}\n```\nFilename: src/front_of_house.rs\n```rust\npub mod hosting {\n pub fn add_to_waitlist() {}\n}\n```\n\n## profile\n```\ncargo build ## dev: [unoptimized + debuginfo]\ncargo build --release ## [optimized]\n```\n- config\n```\n[profile.dev]\nopt-level = 0\n\n[profile.release]\nopt-level = 3\n\n# profile for the wasm example (from project ethers-rs)\n[profile.release.package.ethers-wasm]\nopt-level = \"s\" # Tell `rustc` to optimize for small code size.\n```\n","slug":"rust/rust-08-project-management","published":1,"updated":"2023-05-03T07:41:48.892Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wps000p6hsj78b40thl","content":"

basic concepts

    \n
  • Packages: A Cargo feature that lets you build, test, and share crates
    • \n
  • Crates: A tree of modules that produces a library or executable
    • \n
  • Modules and use: Let you control the organization, scope, and privacy of paths
    • \n
  • Paths: A way of naming an item, such as a struct, function, or module
    • \n
\n

package

    \n
  1. one package can only has one library Crate. default is src/lib.rs
    2. \n
  2. one package can have multiple binary Crates (under src/bin). default src/main.rs
    3. \n
  3. one package has at least one crate，no matter lib or bin。
    4. \n
  4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)
    1
    cargo new my-project --lib  ## create a library project
    5. \n
\n

crate

    \n
  • binary or library
    • \n
  • The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate
    • \n
\n

module

    \n
  • keyword mod
    • \n
  • module could be nested。
    • \n
  • module includes(struct、enum、const、trait、func, etc)
    • \n
\n

path

    \n
  • absolute path: use crate name or crate
    • \n
  • relative path:，use self, super etc
    1
    2
    3
    4
    5
    6
    7
    8
    fn serve_order() {}
    mod back_of_house {
        fn fix_incorrect_order() {
            cook_order();
            super::serve_order();
        }
        fn cook_order() {}
    }
    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    mod front_of_house {
        pub mod hosting {
            pub fn add_to_waitlist() {}
        }
    }
    pub fn eat_at_restaurant() {
        // Absolute path
        crate::front_of_house::hosting::add_to_waitlist();
        // Relative path
        front_of_house::hosting::add_to_waitlist();
    }
    • \n
\n

Bringing Paths into Scope with the use Keyword

1
2
3
4
5
6
7
8
9
use std::io;
use std::io::Write;
use std::io:: { self, Write} ;
use std::{cmp::Ordering, io};
use std::fmt::Result;
use std::io::Result as IoResult;
use std::collections::*;

pub use crate::front_of_house::hosting; // re-exporting names with pub use
\n

Separating Modules into Different Files

Filename: src/lib.rs

\n
1
2
3
4
5
6
7
8
9
mod front_of_house;

pub use crate::front_of_house::hosting;

pub fn eat_at_restaurant() {
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
}
\n

Filename: src/front_of_house.rs

\n
1
2
3
pub mod hosting {
    pub fn add_to_waitlist() {}
}
\n\n

profile

1
2
cargo build ## dev: [unoptimized + debuginfo]
cargo build --release ## [optimized]
\n
    \n
  • config
    1
    2
    3
    4
    5
    6
    7
    8
    9
    [profile.dev]
    opt-level = 0

    [profile.release]
    opt-level = 3

    # profile for the wasm example (from project ethers-rs)
    [profile.release.package.ethers-wasm]
    opt-level = "s"  # Tell `rustc` to optimize for small code size.
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

basic concepts

    \n
  • Packages: A Cargo feature that lets you build, test, and share crates
    • \n
  • Crates: A tree of modules that produces a library or executable
    • \n
  • Modules and use: Let you control the organization, scope, and privacy of paths
    • \n
  • Paths: A way of naming an item, such as a struct, function, or module
    • \n
\n

package

    \n
  1. one package can only has one library Crate. default is src/lib.rs
    2. \n
  2. one package can have multiple binary Crates (under src/bin). default src/main.rs
    3. \n
  3. one package has at least one crate，no matter lib or bin。
    4. \n
  4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)
    1
    cargo new my-project --lib  ## create a library project
    5. \n
\n

crate

    \n
  • binary or library
    • \n
  • The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate
    • \n
\n

module

    \n
  • keyword mod
    • \n
  • module could be nested。
    • \n
  • module includes(struct、enum、const、trait、func, etc)
    • \n
\n

path

    \n
  • absolute path: use crate name or crate
    • \n
  • relative path:，use self, super etc
    1
    2
    3
    4
    5
    6
    7
    8
    fn serve_order() {}
    mod back_of_house {
        fn fix_incorrect_order() {
            cook_order();
            super::serve_order();
        }
        fn cook_order() {}
    }
    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    mod front_of_house {
        pub mod hosting {
            pub fn add_to_waitlist() {}
        }
    }
    pub fn eat_at_restaurant() {
        // Absolute path
        crate::front_of_house::hosting::add_to_waitlist();
        // Relative path
        front_of_house::hosting::add_to_waitlist();
    }
    • \n
\n

Bringing Paths into Scope with the use Keyword

1
2
3
4
5
6
7
8
9
use std::io;
use std::io::Write;
use std::io:: { self, Write} ;
use std::{cmp::Ordering, io};
use std::fmt::Result;
use std::io::Result as IoResult;
use std::collections::*;

pub use crate::front_of_house::hosting; // re-exporting names with pub use
\n

Separating Modules into Different Files

Filename: src/lib.rs

\n
1
2
3
4
5
6
7
8
9
mod front_of_house;

pub use crate::front_of_house::hosting;

pub fn eat_at_restaurant() {
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
}
\n

Filename: src/front_of_house.rs

\n
1
2
3
pub mod hosting {
    pub fn add_to_waitlist() {}
}
\n\n

profile

1
2
cargo build ## dev: [unoptimized + debuginfo]
cargo build --release ## [optimized]
\n
    \n
  • config
    1
    2
    3
    4
    5
    6
    7
    8
    9
    [profile.dev]
    opt-level = 0

    [profile.release]
    opt-level = 3

    # profile for the wasm example (from project ethers-rs)
    [profile.release.package.ethers-wasm]
    opt-level = "s"  # Tell `rustc` to optimize for small code size.
    • \n
\n"},{"title":"rust memory layout","date":"2022-10-25T02:54:13.000Z","_content":"\n\n## trait object\na reference to a trait type (or Box) is called trait object\n\n### two ways convert concrete type to trait object\n1. assigning to variable\n```rust\nuse std::io::Write;\n\nlet mut buffer: Vec = vec![];\nlet w: &mut dyn Write = &mut buffer;\n```\n2. pass a concrete type as a argument to a function\n```rust\nfn main() {\n let mut buffer: Vec = vec![];\n writer(&mut buffer);\n}\n\nfn writer(w: &mut dyn Write) {\n // ...\n}\n```\nin both ways, buffer is converted to a trait object implements Write\n\nin memory, a trait object (in the example, w) is a fat point consists two pointers\n![trait_object_memory](/images/rust/memory/trait_object_memory.png)\nthe vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a \"Writer\"\n\n\n## references\n- https://www.youtube.com/watch?v=rDoqT-a6UFg","source":"_posts/rust/rust-05-memory-layout.md","raw":"---\ntitle: rust memory layout\ndate: 2022-10-25 10:54:13\ntags: [rust]\n---\n\n\n## trait object\na reference to a trait type (or Box) is called trait object\n\n### two ways convert concrete type to trait object\n1. assigning to variable\n```rust\nuse std::io::Write;\n\nlet mut buffer: Vec = vec![];\nlet w: &mut dyn Write = &mut buffer;\n```\n2. pass a concrete type as a argument to a function\n```rust\nfn main() {\n let mut buffer: Vec = vec![];\n writer(&mut buffer);\n}\n\nfn writer(w: &mut dyn Write) {\n // ...\n}\n```\nin both ways, buffer is converted to a trait object implements Write\n\nin memory, a trait object (in the example, w) is a fat point consists two pointers\n![trait_object_memory](/images/rust/memory/trait_object_memory.png)\nthe vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a \"Writer\"\n\n\n## references\n- https://www.youtube.com/watch?v=rDoqT-a6UFg","slug":"rust/rust-05-memory-layout","published":1,"updated":"2023-05-03T02:57:52.719Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpt000s6hsjgtjq4nac","content":"

trait object

a reference to a trait type (or Box) is called trait object

\n

two ways convert concrete type to trait object

    \n
  1. assigning to variable
    1
    2
    3
    4
    use std::io::Write;

    let mut buffer: Vec<u8> = vec![];
    let w: &mut dyn Write = &mut buffer;
    2. \n
  2. pass a concrete type as a argument to a function
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
       let mut buffer: Vec<u8> = vec![];
       writer(&mut buffer);
    }

    fn writer(w: &mut dyn Write) {
        // ...
    }
    \nin both ways, buffer is converted to a trait object implements Write
    3. \n
\n

in memory, a trait object (in the example, w) is a fat point consists two pointers
\"trait_object_memory\"
the vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a “Writer”

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

trait object

a reference to a trait type (or Box) is called trait object

\n

two ways convert concrete type to trait object

    \n
  1. assigning to variable
    1
    2
    3
    4
    use std::io::Write;

    let mut buffer: Vec<u8> = vec![];
    let w: &mut dyn Write = &mut buffer;
    2. \n
  2. pass a concrete type as a argument to a function
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
       let mut buffer: Vec<u8> = vec![];
       writer(&mut buffer);
    }

    fn writer(w: &mut dyn Write) {
        // ...
    }
    \nin both ways, buffer is converted to a trait object implements Write
    3. \n
\n

in memory, a trait object (in the example, w) is a fat point consists two pointers
\"trait_object_memory\"
the vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a “Writer”

\n

references

\n"},{"title":"rust concurrency","date":"2023-06-01T14:04:38.000Z","_content":"\n## Send and Sync\n- A type is Send if it is safe to send it to another thread.\n- A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).\n- raw pointers are neither Send nor Sync (because they have no safety guards).\n- UnsafeCell isn't Sync (and therefore Cell and RefCell aren't).\n- Rc isn't Send or Sync (because the refcount is shared and unsynchronized).\n\nTypes that aren't automatically derived can simply implement them if desired:\n```rust\nstruct MyBox(*mut u8);\n\nunsafe impl Send for MyBox {}\nunsafe impl Sync for MyBox {}\n```\none can also unimplement Send and Sync:\n```rust\n#![feature(negative_impls)]\n\n// I have some magic semantics for some synchronization primitive!\nstruct SpecialThreadToken(u8);\n\nimpl !Send for SpecialThreadToken {}\nimpl !Sync for SpecialThreadToken {}\n```\n\n## reference\n- [rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html)","source":"_posts/rust/rust-10-concurrency.md","raw":"---\ntitle: rust concurrency\ndate: 2023-06-01 22:04:38\ntags: [rust]\n---\n\n## Send and Sync\n- A type is Send if it is safe to send it to another thread.\n- A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).\n- raw pointers are neither Send nor Sync (because they have no safety guards).\n- UnsafeCell isn't Sync (and therefore Cell and RefCell aren't).\n- Rc isn't Send or Sync (because the refcount is shared and unsynchronized).\n\nTypes that aren't automatically derived can simply implement them if desired:\n```rust\nstruct MyBox(*mut u8);\n\nunsafe impl Send for MyBox {}\nunsafe impl Sync for MyBox {}\n```\none can also unimplement Send and Sync:\n```rust\n#![feature(negative_impls)]\n\n// I have some magic semantics for some synchronization primitive!\nstruct SpecialThreadToken(u8);\n\nimpl !Send for SpecialThreadToken {}\nimpl !Sync for SpecialThreadToken {}\n```\n\n## reference\n- [rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html)","slug":"rust/rust-10-concurrency","published":1,"updated":"2023-07-02T07:42:23.897Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpt000u6hsj6fqz3nu7","content":"

Send and Sync

    \n
  • A type is Send if it is safe to send it to another thread.
    • \n
  • A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).
    • \n
  • raw pointers are neither Send nor Sync (because they have no safety guards).
    • \n
  • UnsafeCell isn’t Sync (and therefore Cell and RefCell aren’t).
    • \n
  • Rc isn’t Send or Sync (because the refcount is shared and unsynchronized).
    • \n
\n

Types that aren’t automatically derived can simply implement them if desired:

\n
1
2
3
4
struct MyBox(*mut u8);

unsafe impl Send for MyBox {}
unsafe impl Sync for MyBox {}
\n

one can also unimplement Send and Sync:

\n
1
2
3
4
5
6
7
#![feature(negative_impls)]

// I have some magic semantics for some synchronization primitive!
struct SpecialThreadToken(u8);

impl !Send for SpecialThreadToken {}
impl !Sync for SpecialThreadToken {}
\n\n

reference

\n","site":{"data":{}},"excerpt":"","more":"

Send and Sync

    \n
  • A type is Send if it is safe to send it to another thread.
    • \n
  • A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).
    • \n
  • raw pointers are neither Send nor Sync (because they have no safety guards).
    • \n
  • UnsafeCell isn’t Sync (and therefore Cell and RefCell aren’t).
    • \n
  • Rc isn’t Send or Sync (because the refcount is shared and unsynchronized).
    • \n
\n

Types that aren’t automatically derived can simply implement them if desired:

\n
1
2
3
4
struct MyBox(*mut u8);

unsafe impl Send for MyBox {}
unsafe impl Sync for MyBox {}
\n

one can also unimplement Send and Sync:

\n
1
2
3
4
5
6
7
#![feature(negative_impls)]

// I have some magic semantics for some synchronization primitive!
struct SpecialThreadToken(u8);

impl !Send for SpecialThreadToken {}
impl !Sync for SpecialThreadToken {}
\n\n

reference

\n"},{"title":"rust async","date":"2023-01-13T09:17:10.000Z","_content":" \n\n## I/O\nI/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口\n\n\n### I/O 模型\n```plantuml\n@startmindmap\n+ **I/O模型**\n++ 同步I/O\n'tag::details[]\n+++_ 阻塞I/O (BIO)\n+++_ 非阻塞I/O (NIO)\n+++_ I/O多路复用\n+++_ 信号驱动I/O\n'end::details[]\n++ 异步I/O\n'tag::details[]\n+++_ linux (AIO, io_uring)\n+++_ windows (IOCP)\n'end::details[]\n@endmindmap\n```\n#### 同步阻塞\n- 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；\n- 此时用户线程阻塞，等待内核将数据准备好；\n- 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。\n\n#### 同步非阻塞\n- 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；\n- 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；\n- 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。\n\n#### 同步多路复用\n- 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）\n- 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；\n\n#### 异步I/O\n- 用户线程进行aio_read，进行系统调用切换到内核；\n- 内核立即返回，并不会阻塞用户线程；\n- 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。\n\n### 流程图\n#### 同步blocking I/O\n\n```plantuml\n@startuml Test Diagram\n\nparticipant \"application\" as app\nparticipant \"kernel\" as kernel\n\nactivate app\nactivate kernel\napp -> kernel: syscall: Read recvfrom\nkernel -> kernel: wait for data (no datagram ready)\nkernel -> kernel: copy datagram to user (datagram ready)\nkernel -> app: return\n@enduml\n```\n\n#### I/O多路复用\n\n### 异步编程\n\n\n## the Future Trait\nA Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this\n\n```rust\n\ntrait SimpleFuture {\n type Output;\n fn poll(&mut self, wake: fn()) -> Poll;\n}\n\nenum Poll {\n Ready(T),\n Pending,\n}\n```\n\nFor example, consider the case where we want to read from a socket that may or may not have data available already.\n```rust\npub struct SocketRead<'a> {\n socket: &'a Socket,\n}\n\nimpl SimpleFuture for SocketRead<'_> {\n type Output = Vec;\n\n fn poll(&mut self, wake: fn()) -> Poll {\n if self.socket.has_data_to_read() {\n // The socket has data -- read it into a buffer and return it.\n Poll::Ready(self.socket.read_buf())\n } else {\n // The socket does not yet have data.\n //\n // Arrange for `wake` to be called once data is available.\n // When data becomes available, `wake` will be called, and the\n // user of this `Future` will know to call `poll` again and\n // receive data.\n self.socket.set_readable_callback(wake);\n Poll::Pending\n }\n }\n}\n\n```\n\n the real Future trait and how it is different\n```rust\ntrait Future {\n type Output;\n fn poll(\n // Note the change from `&mut self` to `Pin<&mut Self>`:\n self: Pin<&mut Self>,\n // and the change from `wake: fn()` to `cx: &mut Context<'_>`:\n cx: &mut Context<'_>,\n ) -> Poll;\n}\n\n```\nThe first change you'll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We'll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.\n\nSecondly, wake: fn() has changed to &mut Context<'_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can't store any data about which Future called wake.\n\nIn a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.\n\n\n## task wakeups with Waker\nWaker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.\n\n## referencs\n[csdn blog](https://blog.csdn.net/XMJYever/article/details/111560976)\n","source":"_posts/rust/rust-async.md","raw":"---\ntitle: rust async\ndate: 2023-01-13 17:17:10\ntags: [rust]\n--- \n\n## I/O\nI/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口\n\n\n### I/O 模型\n```plantuml\n@startmindmap\n+ **I/O模型**\n++ 同步I/O\n'tag::details[]\n+++_ 阻塞I/O (BIO)\n+++_ 非阻塞I/O (NIO)\n+++_ I/O多路复用\n+++_ 信号驱动I/O\n'end::details[]\n++ 异步I/O\n'tag::details[]\n+++_ linux (AIO, io_uring)\n+++_ windows (IOCP)\n'end::details[]\n@endmindmap\n```\n#### 同步阻塞\n- 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；\n- 此时用户线程阻塞，等待内核将数据准备好；\n- 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。\n\n#### 同步非阻塞\n- 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；\n- 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；\n- 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。\n\n#### 同步多路复用\n- 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）\n- 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；\n\n#### 异步I/O\n- 用户线程进行aio_read，进行系统调用切换到内核；\n- 内核立即返回，并不会阻塞用户线程；\n- 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。\n\n### 流程图\n#### 同步blocking I/O\n\n```plantuml\n@startuml Test Diagram\n\nparticipant \"application\" as app\nparticipant \"kernel\" as kernel\n\nactivate app\nactivate kernel\napp -> kernel: syscall: Read recvfrom\nkernel -> kernel: wait for data (no datagram ready)\nkernel -> kernel: copy datagram to user (datagram ready)\nkernel -> app: return\n@enduml\n```\n\n#### I/O多路复用\n\n### 异步编程\n\n\n## the Future Trait\nA Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this\n\n```rust\n\ntrait SimpleFuture {\n type Output;\n fn poll(&mut self, wake: fn()) -> Poll;\n}\n\nenum Poll {\n Ready(T),\n Pending,\n}\n```\n\nFor example, consider the case where we want to read from a socket that may or may not have data available already.\n```rust\npub struct SocketRead<'a> {\n socket: &'a Socket,\n}\n\nimpl SimpleFuture for SocketRead<'_> {\n type Output = Vec;\n\n fn poll(&mut self, wake: fn()) -> Poll {\n if self.socket.has_data_to_read() {\n // The socket has data -- read it into a buffer and return it.\n Poll::Ready(self.socket.read_buf())\n } else {\n // The socket does not yet have data.\n //\n // Arrange for `wake` to be called once data is available.\n // When data becomes available, `wake` will be called, and the\n // user of this `Future` will know to call `poll` again and\n // receive data.\n self.socket.set_readable_callback(wake);\n Poll::Pending\n }\n }\n}\n\n```\n\n the real Future trait and how it is different\n```rust\ntrait Future {\n type Output;\n fn poll(\n // Note the change from `&mut self` to `Pin<&mut Self>`:\n self: Pin<&mut Self>,\n // and the change from `wake: fn()` to `cx: &mut Context<'_>`:\n cx: &mut Context<'_>,\n ) -> Poll;\n}\n\n```\nThe first change you'll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We'll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.\n\nSecondly, wake: fn() has changed to &mut Context<'_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can't store any data about which Future called wake.\n\nIn a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.\n\n\n## task wakeups with Waker\nWaker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.\n\n## referencs\n[csdn blog](https://blog.csdn.net/XMJYever/article/details/111560976)\n","slug":"rust/rust-async","published":1,"updated":"2023-05-03T09:33:13.753Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpu000x6hsj51636y2x","content":"

I/O

I/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口

\n

I/O 模型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
@startmindmap
+ **I/O模型**
++ 同步I/O
'tag::details[]
+++_ 阻塞I/O (BIO)
+++_ 非阻塞I/O (NIO)
+++_ I/O多路复用
+++_ 信号驱动I/O
'end::details[]
++ 异步I/O
'tag::details[]
+++_ linux (AIO, io_uring)
+++_ windows (IOCP)
'end::details[]
@endmindmap
\n

同步阻塞

    \n
  • 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；
    • \n
  • 此时用户线程阻塞，等待内核将数据准备好；
    • \n
  • 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。
    • \n
\n

同步非阻塞

    \n
  • 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；
    • \n
  • 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；
    • \n
  • 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。
    • \n
\n

同步多路复用

    \n
  • 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）
    • \n
  • 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；
    • \n
\n

异步I/O

    \n
  • 用户线程进行aio_read，进行系统调用切换到内核；
    • \n
  • 内核立即返回，并不会阻塞用户线程；
    • \n
  • 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。
    • \n
\n

流程图

同步blocking I/O

1
2
3
4
5
6
7
8
9
10
11
12
@startuml Test Diagram

participant "application" as app
participant "kernel" as kernel

activate app
activate kernel
app -> kernel: syscall: Read recvfrom
kernel -> kernel: wait for data (no datagram ready)
kernel -> kernel: copy datagram to user (datagram ready)
kernel -> app: return
@enduml
\n\n

I/O多路复用

异步编程

the Future Trait

A Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this

\n
1
2
3
4
5
6
7
8
9
10

trait SimpleFuture {
    type Output;
    fn poll(&mut self, wake: fn()) -> Poll<Self::Output>;
}

enum Poll<T> {
    Ready(T),
    Pending,
}
\n\n

For example, consider the case where we want to read from a socket that may or may not have data available already.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
pub struct SocketRead<'a> {
    socket: &'a Socket,
}

impl SimpleFuture for SocketRead<'_> {
    type Output = Vec<u8>;

    fn poll(&mut self, wake: fn()) -> Poll<Self::Output> {
        if self.socket.has_data_to_read() {
            // The socket has data -- read it into a buffer and return it.
            Poll::Ready(self.socket.read_buf())
        } else {
            // The socket does not yet have data.
            //
            // Arrange for `wake` to be called once data is available.
            // When data becomes available, `wake` will be called, and the
            // user of this `Future` will know to call `poll` again and
            // receive data.
            self.socket.set_readable_callback(wake);
            Poll::Pending
        }
    }
}

\n\n

the real Future trait and how it is different

\n
1
2
3
4
5
6
7
8
9
10
trait Future {
    type Output;
    fn poll(
        // Note the change from `&mut self` to `Pin<&mut Self>`:
        self: Pin<&mut Self>,
        // and the change from `wake: fn()` to `cx: &mut Context<'_>`:
        cx: &mut Context<'_>,
    ) -> Poll<Self::Output>;
}

\n

The first change you’ll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We’ll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.

\n

Secondly, wake: fn() has changed to &mut Context<’_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can’t store any data about which Future called wake.

\n

In a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.

\n

task wakeups with Waker

Waker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.

\n

referencs

csdn blog

\n","site":{"data":{}},"excerpt":"","more":"

I/O

I/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口

\n

I/O 模型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
@startmindmap
+ **I/O模型**
++ 同步I/O
'tag::details[]
+++_ 阻塞I/O (BIO)
+++_ 非阻塞I/O (NIO)
+++_ I/O多路复用
+++_ 信号驱动I/O
'end::details[]
++ 异步I/O
'tag::details[]
+++_ linux (AIO, io_uring)
+++_ windows (IOCP)
'end::details[]
@endmindmap
\n

同步阻塞

    \n
  • 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；
    • \n
  • 此时用户线程阻塞，等待内核将数据准备好；
    • \n
  • 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。
    • \n
\n

同步非阻塞

    \n
  • 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；
    • \n
  • 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；
    • \n
  • 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。
    • \n
\n

同步多路复用

    \n
  • 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）
    • \n
  • 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；
    • \n
\n

异步I/O

    \n
  • 用户线程进行aio_read，进行系统调用切换到内核；
    • \n
  • 内核立即返回，并不会阻塞用户线程；
    • \n
  • 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。
    • \n
\n

流程图

同步blocking I/O

1
2
3
4
5
6
7
8
9
10
11
12
@startuml Test Diagram

participant "application" as app
participant "kernel" as kernel

activate app
activate kernel
app -> kernel: syscall: Read recvfrom
kernel -> kernel: wait for data (no datagram ready)
kernel -> kernel: copy datagram to user (datagram ready)
kernel -> app: return
@enduml
\n\n

I/O多路复用

异步编程

the Future Trait

A Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this

\n
1
2
3
4
5
6
7
8
9
10

trait SimpleFuture {
    type Output;
    fn poll(&mut self, wake: fn()) -> Poll<Self::Output>;
}

enum Poll<T> {
    Ready(T),
    Pending,
}
\n\n

For example, consider the case where we want to read from a socket that may or may not have data available already.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
pub struct SocketRead<'a> {
    socket: &'a Socket,
}

impl SimpleFuture for SocketRead<'_> {
    type Output = Vec<u8>;

    fn poll(&mut self, wake: fn()) -> Poll<Self::Output> {
        if self.socket.has_data_to_read() {
            // The socket has data -- read it into a buffer and return it.
            Poll::Ready(self.socket.read_buf())
        } else {
            // The socket does not yet have data.
            //
            // Arrange for `wake` to be called once data is available.
            // When data becomes available, `wake` will be called, and the
            // user of this `Future` will know to call `poll` again and
            // receive data.
            self.socket.set_readable_callback(wake);
            Poll::Pending
        }
    }
}

\n\n

the real Future trait and how it is different

\n
1
2
3
4
5
6
7
8
9
10
trait Future {
    type Output;
    fn poll(
        // Note the change from `&mut self` to `Pin<&mut Self>`:
        self: Pin<&mut Self>,
        // and the change from `wake: fn()` to `cx: &mut Context<'_>`:
        cx: &mut Context<'_>,
    ) -> Poll<Self::Output>;
}

\n

The first change you’ll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We’ll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.

\n

Secondly, wake: fn() has changed to &mut Context<’_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can’t store any data about which Future called wake.

\n

In a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.

\n

task wakeups with Waker

Waker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.

\n

referencs

csdn blog

\n"},{"title":"rust cargo all in one","date":"2022-12-06T09:05:07.000Z","_content":"\n## useful cmd\n```\ncargo new ${crate_name} --lib ## create a lib crate\ncargo build --verbose ## print out each rustc invocation\n```\n\n## specifying dependencies\n- specifying dependencies from crates.io\n```toml\n[dependencies]\ntime = \"0.1.12\"\n```\n- specifying dependencies from other registries\n```toml\n[dependencies]\nsome-crate = { version = \"1.0\", registry = \"my-registry\" }\n```\n- specifying dependencies form git repositories\n```toml\n[dependencies]\nregex = { git = \"https://github.com/rust-lang/regex.git\" }\n```\n- path dependencies\n```toml\n[dependencies]\nhello_utils = { path = \"hello_utils\" }\n```\n- platform specific dependencies\n```toml\n[target.'cfg(unix)'.dependencies]\nopenssl = \"1.0.1\"\n\n[target.'cfg(target_arch = \"x86\")'.dependencies]\nnative-i686 = { path = \"native/i686\" }\n```\nLike with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.\nIf you want to know which cfg targets are available on your platform, run ```rustc --print=cfg``` from the command line.\nIf you want to know which cfg targets are available for another platform, such as 64-bit Windows, run ```rustc --print=cfg --target=x86_64-pc-windows-msvc```\n- custom target specifications\n```toml\n[target.bar.dependencies]\nwinhttp = \"0.4.0\"\n\n[target.my-special-i686-platform.dependencies]\nopenssl = \"1.0.1\"\nnative = { path = \"native/i686\" }\n```\n- development dependencies\n [dev-dependencies]\n Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.\n These dependencies are not propagated to other packages which depend on this package.\n ```toml\n [target.'cfg(unix)'.dev-dependencies]\n mio = \"0.0.1\"\n ```\n\n- build dependencies\n ```toml\n [build-dependencies]\n cc = \"1.0.3\"\n ```\n The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.\n- choosing features\n ```toml\n [dependencies.awesome]\n version = \"1.3.5\"\n default-features = false # do not include the default features, and optionally\n # cherry-pick individual features\n features = [\"secure-password\", \"civet\"]\n ```\n- renaming dependencies in Cargo.toml\n When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it's published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.\n (more to be found in original book)\n\n## references\n[cargo book](https://doc.rust-lang.org/cargo/)","source":"_posts/rust/rust-cargo-all-in-one.md","raw":"---\ntitle: rust cargo all in one\ndate: 2022-12-06 17:05:07\ntags: [rust]\n---\n\n## useful cmd\n```\ncargo new ${crate_name} --lib ## create a lib crate\ncargo build --verbose ## print out each rustc invocation\n```\n\n## specifying dependencies\n- specifying dependencies from crates.io\n```toml\n[dependencies]\ntime = \"0.1.12\"\n```\n- specifying dependencies from other registries\n```toml\n[dependencies]\nsome-crate = { version = \"1.0\", registry = \"my-registry\" }\n```\n- specifying dependencies form git repositories\n```toml\n[dependencies]\nregex = { git = \"https://github.com/rust-lang/regex.git\" }\n```\n- path dependencies\n```toml\n[dependencies]\nhello_utils = { path = \"hello_utils\" }\n```\n- platform specific dependencies\n```toml\n[target.'cfg(unix)'.dependencies]\nopenssl = \"1.0.1\"\n\n[target.'cfg(target_arch = \"x86\")'.dependencies]\nnative-i686 = { path = \"native/i686\" }\n```\nLike with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.\nIf you want to know which cfg targets are available on your platform, run ```rustc --print=cfg``` from the command line.\nIf you want to know which cfg targets are available for another platform, such as 64-bit Windows, run ```rustc --print=cfg --target=x86_64-pc-windows-msvc```\n- custom target specifications\n```toml\n[target.bar.dependencies]\nwinhttp = \"0.4.0\"\n\n[target.my-special-i686-platform.dependencies]\nopenssl = \"1.0.1\"\nnative = { path = \"native/i686\" }\n```\n- development dependencies\n [dev-dependencies]\n Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.\n These dependencies are not propagated to other packages which depend on this package.\n ```toml\n [target.'cfg(unix)'.dev-dependencies]\n mio = \"0.0.1\"\n ```\n\n- build dependencies\n ```toml\n [build-dependencies]\n cc = \"1.0.3\"\n ```\n The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.\n- choosing features\n ```toml\n [dependencies.awesome]\n version = \"1.3.5\"\n default-features = false # do not include the default features, and optionally\n # cherry-pick individual features\n features = [\"secure-password\", \"civet\"]\n ```\n- renaming dependencies in Cargo.toml\n When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it's published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.\n (more to be found in original book)\n\n## references\n[cargo book](https://doc.rust-lang.org/cargo/)","slug":"rust/rust-cargo-all-in-one","published":1,"updated":"2023-05-03T10:04:18.682Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpu000z6hsjb7jy64m2","content":"

useful cmd

1
2
cargo new ${crate_name} --lib ## create a lib crate
cargo build --verbose ## print out each rustc invocation
\n\n

specifying dependencies

    \n

  • specifying dependencies from crates.io

    \n
    1
    2
    [dependencies]
    time = "0.1.12"
    • \n

  • specifying dependencies from other registries

    \n
    1
    2
    [dependencies]
    some-crate = { version = "1.0", registry = "my-registry" }
    • \n

  • specifying dependencies form git repositories

    \n
    1
    2
    [dependencies]
    regex = { git = "https://github.com/rust-lang/regex.git" }
    • \n

  • path dependencies

    \n
    1
    2
    [dependencies]
    hello_utils = { path = "hello_utils" }
    • \n

  • platform specific dependencies

    \n
    1
    2
    3
    4
    5
    [target.'cfg(unix)'.dependencies]
    openssl = "1.0.1"

    [target.'cfg(target_arch = "x86")'.dependencies]
    native-i686 = { path = "native/i686" }
    \n

    Like with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.
    If you want to know which cfg targets are available on your platform, run rustc --print=cfg from the command line.
    If you want to know which cfg targets are available for another platform, such as 64-bit Windows, run rustc --print=cfg --target=x86_64-pc-windows-msvc

    \n
    • \n

  • custom target specifications

    \n
    1
    2
    3
    4
    5
    6
    [target.bar.dependencies]
    winhttp = "0.4.0"

    [target.my-special-i686-platform.dependencies]
    openssl = "1.0.1"
    native = { path = "native/i686" }
    • \n

  • development dependencies
    [dev-dependencies]
    Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.
    These dependencies are not propagated to other packages which depend on this package.

    \n 
    1
    2
    [target.'cfg(unix)'.dev-dependencies]
    mio = "0.0.1"
    \n
    • \n

  • build dependencies

    \n 
    1
    2
    [build-dependencies]
    cc = "1.0.3"
    \n

    The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.

    \n
    • \n

  • choosing features

    \n 
    1
    2
    3
    4
    5
    [dependencies.awesome]
    version = "1.3.5"
    default-features = false # do not include the default features, and optionally
                            # cherry-pick individual features
    features = ["secure-password", "civet"]
    • \n

  • renaming dependencies in Cargo.toml
    When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it’s published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.
    (more to be found in original book)

    \n
    • \n
\n

references

cargo book

\n","site":{"data":{}},"excerpt":"","more":"

useful cmd

1
2
cargo new ${crate_name} --lib ## create a lib crate
cargo build --verbose ## print out each rustc invocation
\n\n

specifying dependencies

    \n

  • specifying dependencies from crates.io

    \n
    1
    2
    [dependencies]
    time = "0.1.12"
    • \n

  • specifying dependencies from other registries

    \n
    1
    2
    [dependencies]
    some-crate = { version = "1.0", registry = "my-registry" }
    • \n

  • specifying dependencies form git repositories

    \n
    1
    2
    [dependencies]
    regex = { git = "https://github.com/rust-lang/regex.git" }
    • \n

  • path dependencies

    \n
    1
    2
    [dependencies]
    hello_utils = { path = "hello_utils" }
    • \n

  • platform specific dependencies

    \n
    1
    2
    3
    4
    5
    [target.'cfg(unix)'.dependencies]
    openssl = "1.0.1"

    [target.'cfg(target_arch = "x86")'.dependencies]
    native-i686 = { path = "native/i686" }
    \n

    Like with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.
    If you want to know which cfg targets are available on your platform, run rustc --print=cfg from the command line.
    If you want to know which cfg targets are available for another platform, such as 64-bit Windows, run rustc --print=cfg --target=x86_64-pc-windows-msvc

    \n
    • \n

  • custom target specifications

    \n
    1
    2
    3
    4
    5
    6
    [target.bar.dependencies]
    winhttp = "0.4.0"

    [target.my-special-i686-platform.dependencies]
    openssl = "1.0.1"
    native = { path = "native/i686" }
    • \n

  • development dependencies
    [dev-dependencies]
    Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.
    These dependencies are not propagated to other packages which depend on this package.

    \n 
    1
    2
    [target.'cfg(unix)'.dev-dependencies]
    mio = "0.0.1"
    \n
    • \n

  • build dependencies

    \n 
    1
    2
    [build-dependencies]
    cc = "1.0.3"
    \n

    The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.

    \n
    • \n

  • choosing features

    \n 
    1
    2
    3
    4
    5
    [dependencies.awesome]
    version = "1.3.5"
    default-features = false # do not include the default features, and optionally
                            # cherry-pick individual features
    features = ["secure-password", "civet"]
    • \n

  • renaming dependencies in Cargo.toml
    When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it’s published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.
    (more to be found in original book)

    \n
    • \n
\n

references

cargo book

\n"},{"title":"cargo doc","date":"2022-11-13T07:41:59.000Z","_content":"\n## 文档注释\n### 用于生成文档\n - 使用 ///\n - 支持 Markdown\n - 放置在被说没条目之前\n### 例子\n```rust\n/// adds one to the number given\n/// \n/// # Examples\n/// ```\n/// let arg = 5;\n/// let answer = my_crate::add_one(arg);\n/// \n/// assert_eq!(6, answer);\n/// ```\npub fn add_one(x: i32) -> i32 {\n x + 1;\n}\n```\n### 命令\n```\ncargo doc ## 生成的html放在 target/doc 目录下\ncargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)\n```\n### 常用章节\n- `# Examples`\n- `Panics`: 可能panic的场景\n- `Errors`: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件\n- `Safety`: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提\n\n### 文档注释作为测试\n- 运行cargo test, doc中用# Example标记的实例代码会用来测试运行\n\n### 为包含注释的项添加文档注释\n- 符号: //!\n- 这类注释通常用描述crate和模块:\n - crate root (按惯例 src/lib.rs)\n - 一个模块内，将crate火模块作为一个整体进行记录\n\n## 注释\n//! - 模块级稳定注释, 置于模块头部\n//!! - 模块级稳定注释, 但是和上面注释置于同一行\n\n//! - 模块级稳定注释, 会换行\n\n/*! - 模块级稳定注释 */\n/*!！ - 模块级稳定注释, 和上面同一行 */\n\n// 普通行注释\n/// 行级文档注释\n//// 普通行注释\n\n/* 普通块级注释 */\n/** 会级文档注释 */","source":"_posts/rust/rust-cargo-doc.md","raw":"---\ntitle: cargo doc\ndate: 2022-11-13 15:41:59\ntags: [rust,cargo]\n---\n\n## 文档注释\n### 用于生成文档\n - 使用 ///\n - 支持 Markdown\n - 放置在被说没条目之前\n### 例子\n```rust\n/// adds one to the number given\n/// \n/// # Examples\n/// ```\n/// let arg = 5;\n/// let answer = my_crate::add_one(arg);\n/// \n/// assert_eq!(6, answer);\n/// ```\npub fn add_one(x: i32) -> i32 {\n x + 1;\n}\n```\n### 命令\n```\ncargo doc ## 生成的html放在 target/doc 目录下\ncargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)\n```\n### 常用章节\n- `# Examples`\n- `Panics`: 可能panic的场景\n- `Errors`: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件\n- `Safety`: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提\n\n### 文档注释作为测试\n- 运行cargo test, doc中用# Example标记的实例代码会用来测试运行\n\n### 为包含注释的项添加文档注释\n- 符号: //!\n- 这类注释通常用描述crate和模块:\n - crate root (按惯例 src/lib.rs)\n - 一个模块内，将crate火模块作为一个整体进行记录\n\n## 注释\n//! - 模块级稳定注释, 置于模块头部\n//!! - 模块级稳定注释, 但是和上面注释置于同一行\n\n//! - 模块级稳定注释, 会换行\n\n/*! - 模块级稳定注释 */\n/*!！ - 模块级稳定注释, 和上面同一行 */\n\n// 普通行注释\n/// 行级文档注释\n//// 普通行注释\n\n/* 普通块级注释 */\n/** 会级文档注释 */","slug":"rust/rust-cargo-doc","published":1,"updated":"2023-05-03T09:28:51.353Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpu00126hsjf6vr6ylq","content":"

文档注释

用于生成文档

    \n
  • 使用 ///
    • \n
  • 支持 Markdown
    • \n
  • 放置在被说没条目之前
    • \n
\n

例子

1
2
3
4
5
6
7
8
9
10
11
12
/// adds one to the number given
/// 
/// # Examples
/// ```
/// let arg = 5;
/// let answer = my_crate::add_one(arg);
/// 
/// assert_eq!(6, answer);
/// ```
pub fn add_one(x: i32) -> i32 {
    x + 1;
}
\n

命令

1
2
cargo doc  ## 生成的html放在 target/doc 目录下
cargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)
\n

常用章节

    \n
  • # Examples
    • \n
  • Panics: 可能panic的场景
    • \n
  • Errors: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件
    • \n
  • Safety: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提
    • \n
\n

文档注释作为测试

    \n
  • 运行cargo test, doc中用# Example标记的实例代码会用来测试运行
    • \n
\n

为包含注释的项添加文档注释

    \n
  • 符号: //!
    • \n
  • 这类注释通常用描述crate和模块:
      \n
    • crate root (按惯例 src/lib.rs)
      • \n
    • 一个模块内，将crate火模块作为一个整体进行记录
      • \n
    \n
    • \n
\n

注释

//! - 模块级稳定注释, 置于模块头部
//!! - 模块级稳定注释, 但是和上面注释置于同一行

\n

//! - 模块级稳定注释, 会换行

\n

/*! - 模块级稳定注释 /
/!！ - 模块级稳定注释, 和上面同一行 */

\n

// 普通行注释
/// 行级文档注释
//// 普通行注释

\n

/* 普通块级注释 /
/* 会级文档注释 */

\n","site":{"data":{}},"excerpt":"","more":"

文档注释

用于生成文档

    \n
  • 使用 ///
    • \n
  • 支持 Markdown
    • \n
  • 放置在被说没条目之前
    • \n
\n

例子

1
2
3
4
5
6
7
8
9
10
11
12
/// adds one to the number given
/// 
/// # Examples
/// ```
/// let arg = 5;
/// let answer = my_crate::add_one(arg);
/// 
/// assert_eq!(6, answer);
/// ```
pub fn add_one(x: i32) -> i32 {
    x + 1;
}
\n

命令

1
2
cargo doc  ## 生成的html放在 target/doc 目录下
cargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)
\n

常用章节

    \n
  • # Examples
    • \n
  • Panics: 可能panic的场景
    • \n
  • Errors: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件
    • \n
  • Safety: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提
    • \n
\n

文档注释作为测试

    \n
  • 运行cargo test, doc中用# Example标记的实例代码会用来测试运行
    • \n
\n

为包含注释的项添加文档注释

    \n
  • 符号: //!
    • \n
  • 这类注释通常用描述crate和模块:
      \n
    • crate root (按惯例 src/lib.rs)
      • \n
    • 一个模块内，将crate火模块作为一个整体进行记录
      • \n
    \n
    • \n
\n

注释

//! - 模块级稳定注释, 置于模块头部
//!! - 模块级稳定注释, 但是和上面注释置于同一行

\n

//! - 模块级稳定注释, 会换行

\n

/*! - 模块级稳定注释 /
/!！ - 模块级稳定注释, 和上面同一行 */

\n

// 普通行注释
/// 行级文档注释
//// 普通行注释

\n

/* 普通块级注释 /
/* 会级文档注释 */

\n"},{"title":"rust sugar","date":"2022-11-17T07:47:13.000Z","_content":"\n## formatted print\n```rust\n// Positional arguments can be used\nprintln!(\"{0}, this is {1}. {1}, this is {0}\", \"Alice\", \"Bob\");\n// As can named arguments.\nprintln!(\"{subject} {verb} {object}\",\n object=\"the lazy dog\",\n subject=\"the quick brown fox\",\n verb=\"jumps over\");\n// Different formatting can be invoked by specifying the format character\n// after a `:`.\nprintln!(\"Base 10: {}\", 69420); // 69420\nprintln!(\"Base 2 (binary): {:b}\", 69420); // 10000111100101100\nprintln!(\"Base 8 (octal): {:o}\", 69420); // 207454\nprintln!(\"Base 16 (hexadecimal): {:x}\", 69420); // 10f2c\nprintln!(\"Base 16 (hexadecimal): {:X}\", 69420); // 10F2C\n// You can right-justify text with a specified width. This will\n// output \" 1\". (Four white spaces and a \"1\", for a total width of 5.)\nprintln!(\"{number:>5}\", number=1);\n// You can pad numbers with extra zeroes,\n// and left-adjust by flipping the sign. This will output \"10000\".\nprintln!(\"{number:0<5}\", number=1);\n// You can use named arguments in the format specifier by appending a `$`.\nprintln!(\"{number:0>width$}\", number=1, width=5);\n```\n- [reference](https://doc.rust-lang.org/rust-by-example/hello/print.html)\n\n\n## syntax\n```rust\n/// print to stderr\neprintln!(\"server error: {}\", e);\n\nstruct MyTupleStruct(M);\nlet myTupleStruct = MyTupleStruct::(String::from(\"hello\"));\n\nvec.iter().position()\nvec.iter().find()\nvec.iter().any()\n\n\nstatic mut A: u32 = 0;\n```\n\n## attributes\n```rust\n#![allow(warnings)]\n#[allow(dead_code)]\n#![allow(unused)]\n// Suppress all warnings from casts which overflow.\n#![allow(overflowing_literals)]\n#![allow(unreachable_code)]\n```\n\n## memory\nThe compiler will not rearrange the memory layout\n```rust\n#[repr(C)]\nstruct A {\n a:u8,\n b:u32,\n c:u16\n}\n```\n\n## ptr\n```rust\nNonNull::new_unchecked()\n\n#![feature(new_uninit)]\nlet mut five = Box::::new_uninit();\nlet five = unsafe {\n // Deferred initialization:\n five.as_mut_ptr().write(5);\n five.assume_init()\n};\nassert_eq!(*five, 5)\n\nlet zero = Box::::new_zeroed();\nlet zero = unsafe { zero.assume_init() };\nassert_eq!(*zero, 0)\n\nuse std::alloc::{alloc, Layout};\nunsafe {\n let ptr = alloc(Layout::new::()) as *mut i32;\n ptr.write(5);\n let x = Box::from_raw(ptr);\n}\n```","source":"_posts/rust/rust-sugar.md","raw":"---\ntitle: rust sugar\ndate: 2022-11-17 15:47:13\ntags: [rust]\n---\n\n## formatted print\n```rust\n// Positional arguments can be used\nprintln!(\"{0}, this is {1}. {1}, this is {0}\", \"Alice\", \"Bob\");\n// As can named arguments.\nprintln!(\"{subject} {verb} {object}\",\n object=\"the lazy dog\",\n subject=\"the quick brown fox\",\n verb=\"jumps over\");\n// Different formatting can be invoked by specifying the format character\n// after a `:`.\nprintln!(\"Base 10: {}\", 69420); // 69420\nprintln!(\"Base 2 (binary): {:b}\", 69420); // 10000111100101100\nprintln!(\"Base 8 (octal): {:o}\", 69420); // 207454\nprintln!(\"Base 16 (hexadecimal): {:x}\", 69420); // 10f2c\nprintln!(\"Base 16 (hexadecimal): {:X}\", 69420); // 10F2C\n// You can right-justify text with a specified width. This will\n// output \" 1\". (Four white spaces and a \"1\", for a total width of 5.)\nprintln!(\"{number:>5}\", number=1);\n// You can pad numbers with extra zeroes,\n// and left-adjust by flipping the sign. This will output \"10000\".\nprintln!(\"{number:0<5}\", number=1);\n// You can use named arguments in the format specifier by appending a `$`.\nprintln!(\"{number:0>width$}\", number=1, width=5);\n```\n- [reference](https://doc.rust-lang.org/rust-by-example/hello/print.html)\n\n\n## syntax\n```rust\n/// print to stderr\neprintln!(\"server error: {}\", e);\n\nstruct MyTupleStruct(M);\nlet myTupleStruct = MyTupleStruct::(String::from(\"hello\"));\n\nvec.iter().position()\nvec.iter().find()\nvec.iter().any()\n\n\nstatic mut A: u32 = 0;\n```\n\n## attributes\n```rust\n#![allow(warnings)]\n#[allow(dead_code)]\n#![allow(unused)]\n// Suppress all warnings from casts which overflow.\n#![allow(overflowing_literals)]\n#![allow(unreachable_code)]\n```\n\n## memory\nThe compiler will not rearrange the memory layout\n```rust\n#[repr(C)]\nstruct A {\n a:u8,\n b:u32,\n c:u16\n}\n```\n\n## ptr\n```rust\nNonNull::new_unchecked()\n\n#![feature(new_uninit)]\nlet mut five = Box::::new_uninit();\nlet five = unsafe {\n // Deferred initialization:\n five.as_mut_ptr().write(5);\n five.assume_init()\n};\nassert_eq!(*five, 5)\n\nlet zero = Box::::new_zeroed();\nlet zero = unsafe { zero.assume_init() };\nassert_eq!(*zero, 0)\n\nuse std::alloc::{alloc, Layout};\nunsafe {\n let ptr = alloc(Layout::new::()) as *mut i32;\n ptr.write(5);\n let x = Box::from_raw(ptr);\n}\n```","slug":"rust/rust-sugar","published":1,"updated":"2023-07-11T07:36:27.147Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpv00146hsj0upzhkpn","content":"

formatted print

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// Positional arguments can be used
println!("{0}, this is {1}. {1}, this is {0}", "Alice", "Bob");
// As can named arguments.
println!("{subject} {verb} {object}",
            object="the lazy dog",
            subject="the quick brown fox",
            verb="jumps over");
// Different formatting can be invoked by specifying the format character
// after a `:`.
println!("Base 10:               {}",   69420); // 69420
println!("Base 2 (binary):       {:b}", 69420); // 10000111100101100
println!("Base 8 (octal):        {:o}", 69420); // 207454
println!("Base 16 (hexadecimal): {:x}", 69420); // 10f2c
println!("Base 16 (hexadecimal): {:X}", 69420); // 10F2C
// You can right-justify text with a specified width. This will
// output "    1". (Four white spaces and a "1", for a total width of 5.)
println!("{number:>5}", number=1);
// You can pad numbers with extra zeroes,
// and left-adjust by flipping the sign. This will output "10000".
println!("{number:0<5}", number=1);
// You can use named arguments in the format specifier by appending a `$`.
println!("{number:0>width$}", number=1, width=5);
\n\n

syntax

1
2
3
4
5
6
7
8
9
10
11
12
/// print to stderr
eprintln!("server error: {}", e);

struct MyTupleStruct<M>(M);
let myTupleStruct =  MyTupleStruct::<String>(String::from("hello"));

vec.iter().position()
vec.iter().find()
vec.iter().any()


static mut A: u32 = 0;
\n\n

attributes

1
2
3
4
5
6
#![allow(warnings)]
#[allow(dead_code)]
#![allow(unused)]
// Suppress all warnings from casts which overflow.
#![allow(overflowing_literals)]
#![allow(unreachable_code)]
\n\n

memory

The compiler will not rearrange the memory layout

\n
1
2
3
4
5
6
#[repr(C)]
struct A {
    a:u8,
    b:u32,
    c:u16
}
\n\n

ptr

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
NonNull::new_unchecked()

#![feature(new_uninit)]
let mut five = Box::<u32>::new_uninit();
let five = unsafe {
    // Deferred initialization:
    five.as_mut_ptr().write(5);
    five.assume_init()
};
assert_eq!(*five, 5)

let zero = Box::<u32>::new_zeroed();
let zero = unsafe { zero.assume_init() };
assert_eq!(*zero, 0)

use std::alloc::{alloc, Layout};
unsafe {
    let ptr = alloc(Layout::new::<i32>()) as *mut i32;
    ptr.write(5);
    let x = Box::from_raw(ptr);
}
","site":{"data":{}},"excerpt":"","more":"

formatted print

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// Positional arguments can be used
println!("{0}, this is {1}. {1}, this is {0}", "Alice", "Bob");
// As can named arguments.
println!("{subject} {verb} {object}",
            object="the lazy dog",
            subject="the quick brown fox",
            verb="jumps over");
// Different formatting can be invoked by specifying the format character
// after a `:`.
println!("Base 10:               {}",   69420); // 69420
println!("Base 2 (binary):       {:b}", 69420); // 10000111100101100
println!("Base 8 (octal):        {:o}", 69420); // 207454
println!("Base 16 (hexadecimal): {:x}", 69420); // 10f2c
println!("Base 16 (hexadecimal): {:X}", 69420); // 10F2C
// You can right-justify text with a specified width. This will
// output "    1". (Four white spaces and a "1", for a total width of 5.)
println!("{number:>5}", number=1);
// You can pad numbers with extra zeroes,
// and left-adjust by flipping the sign. This will output "10000".
println!("{number:0<5}", number=1);
// You can use named arguments in the format specifier by appending a `$`.
println!("{number:0>width$}", number=1, width=5);
\n\n

syntax

1
2
3
4
5
6
7
8
9
10
11
12
/// print to stderr
eprintln!("server error: {}", e);

struct MyTupleStruct<M>(M);
let myTupleStruct =  MyTupleStruct::<String>(String::from("hello"));

vec.iter().position()
vec.iter().find()
vec.iter().any()


static mut A: u32 = 0;
\n\n

attributes

1
2
3
4
5
6
#![allow(warnings)]
#[allow(dead_code)]
#![allow(unused)]
// Suppress all warnings from casts which overflow.
#![allow(overflowing_literals)]
#![allow(unreachable_code)]
\n\n

memory

The compiler will not rearrange the memory layout

\n
1
2
3
4
5
6
#[repr(C)]
struct A {
    a:u8,
    b:u32,
    c:u16
}
\n\n

ptr

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
NonNull::new_unchecked()

#![feature(new_uninit)]
let mut five = Box::<u32>::new_uninit();
let five = unsafe {
    // Deferred initialization:
    five.as_mut_ptr().write(5);
    five.assume_init()
};
assert_eq!(*five, 5)

let zero = Box::<u32>::new_zeroed();
let zero = unsafe { zero.assume_init() };
assert_eq!(*zero, 0)

use std::alloc::{alloc, Layout};
unsafe {
    let ptr = alloc(Layout::new::<i32>()) as *mut i32;
    ptr.write(5);
    let x = Box::from_raw(ptr);
}
"},{"title":"rust similar concepts comparison","date":"2022-11-23T07:52:34.000Z","_content":"\n## ref vs &\n`ref` annotates pattern bindings to make them borrow rather than move. It is **not** a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.\nBy default, match statements consume all they can, which can sometimes be a problem, when you don't really need the value to be moved and owned:\n```rust\nlet maybe_name = Some(String::from(\"Alice\"));\n// Using `ref`, the value is borrowed, not moved ...\nmatch maybe_name {\n Some(ref n) => println!(\"Hello, {n}\"),\n _ => println!(\"Hello, world\"),\n}\n// ... so it's available here!\nprintln!(\"Hello again, {}\", maybe_name.unwrap_or(\"world\".into()));\n```\n\n- `&` denotes that your pattern expects a reference to an object. Hence `&` is a part of said pattern: `&Foo` matches different objects than `Foo` does.\n- `ref` indicates that you want a reference to an unpacked value. It is not matched against: `Foo(ref foo)` matches the same objects as `Foo(foo)`.\n\n## Clone vs Copy\n### Copy 的含义\n`Copy` 的全名是 `std::marker::Copy`。`std::marker` 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 `Copy`、`Send`、`Sized`、`Sync`。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。\n\n如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。\n\n一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。\n\n### Copy 的实现条件\n并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。\n\n常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。\n\n我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。\n\n### Clone 的含义\nClone 的全名是 std::clone::Clone。它的完整声明是这样的：\n```rust\npub trait Clone : Sized {\n fn clone(&self) -> Self;\n fn clone_from(&mut self, source: &Self) {\n *self = source.clone()\n }\n}\n```\n它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。\n\nclone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。\n\n虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。\n\n### 自动 derive\n绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：\n\n```rust\n#[derive(Copy, Clone)]\nstruct MyStruct(i32);\n```\n这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。\n\n通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。\n\n目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。\n\n## Cell vs RefCell\n- Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(\"Hello\")).get()会报错\n- Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic\n- 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大\n\n## AsRef vs Borrow\n[WIP]\n","source":"_posts/rust/rust-similar-concepts-comparison.md","raw":"---\ntitle: rust similar concepts comparison\ndate: 2022-11-23 15:52:34\ntags: [rust]\n---\n\n## ref vs &\n`ref` annotates pattern bindings to make them borrow rather than move. It is **not** a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.\nBy default, match statements consume all they can, which can sometimes be a problem, when you don't really need the value to be moved and owned:\n```rust\nlet maybe_name = Some(String::from(\"Alice\"));\n// Using `ref`, the value is borrowed, not moved ...\nmatch maybe_name {\n Some(ref n) => println!(\"Hello, {n}\"),\n _ => println!(\"Hello, world\"),\n}\n// ... so it's available here!\nprintln!(\"Hello again, {}\", maybe_name.unwrap_or(\"world\".into()));\n```\n\n- `&` denotes that your pattern expects a reference to an object. Hence `&` is a part of said pattern: `&Foo` matches different objects than `Foo` does.\n- `ref` indicates that you want a reference to an unpacked value. It is not matched against: `Foo(ref foo)` matches the same objects as `Foo(foo)`.\n\n## Clone vs Copy\n### Copy 的含义\n`Copy` 的全名是 `std::marker::Copy`。`std::marker` 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 `Copy`、`Send`、`Sized`、`Sync`。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。\n\n如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。\n\n一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。\n\n### Copy 的实现条件\n并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。\n\n常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。\n\n我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。\n\n### Clone 的含义\nClone 的全名是 std::clone::Clone。它的完整声明是这样的：\n```rust\npub trait Clone : Sized {\n fn clone(&self) -> Self;\n fn clone_from(&mut self, source: &Self) {\n *self = source.clone()\n }\n}\n```\n它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。\n\nclone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。\n\n虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。\n\n### 自动 derive\n绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：\n\n```rust\n#[derive(Copy, Clone)]\nstruct MyStruct(i32);\n```\n这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。\n\n通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。\n\n目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。\n\n## Cell vs RefCell\n- Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(\"Hello\")).get()会报错\n- Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic\n- 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大\n\n## AsRef vs Borrow\n[WIP]\n","slug":"rust/rust-similar-concepts-comparison","published":1,"updated":"2023-07-09T08:33:27.383Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpv00176hsjd2sh9yph","content":"

ref vs &

ref annotates pattern bindings to make them borrow rather than move. It is not a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.
By default, match statements consume all they can, which can sometimes be a problem, when you don’t really need the value to be moved and owned:

\n
1
2
3
4
5
6
7
8
let maybe_name = Some(String::from("Alice"));
// Using `ref`, the value is borrowed, not moved ...
match maybe_name {
    Some(ref n) => println!("Hello, {n}"),
    _ => println!("Hello, world"),
}
// ... so it's available here!
println!("Hello again, {}", maybe_name.unwrap_or("world".into()));
\n\n
    \n
  • & denotes that your pattern expects a reference to an object. Hence & is a part of said pattern: &Foo matches different objects than Foo does.
    • \n
  • ref indicates that you want a reference to an unpacked value. It is not matched against: Foo(ref foo) matches the same objects as Foo(foo).
    • \n
\n

Clone vs Copy

Copy 的含义

Copy 的全名是 std::marker::Copy。std::marker 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 Copy、Send、Sized、Sync。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。

\n

如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。

\n

一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。

\n

Copy 的实现条件

并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。

\n

常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。

\n

我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。

\n

Clone 的含义

Clone 的全名是 std::clone::Clone。它的完整声明是这样的：

\n
1
2
3
4
5
6
pub trait Clone : Sized {
    fn clone(&self) -> Self;
    fn clone_from(&mut self, source: &Self) {
        *self = source.clone()
    }
}
\n

它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。

\n

clone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。

\n

虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。

\n

自动 derive

绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：

\n
1
2
#[derive(Copy, Clone)]
struct MyStruct(i32);
\n

这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。

\n

通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。

\n

目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。

\n

Cell vs RefCell

    \n
  • Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(“Hello”)).get()会报错
    • \n
  • Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic
    • \n
  • 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大
    • \n
\n

AsRef vs Borrow

[WIP]

\n","site":{"data":{}},"excerpt":"","more":"

ref vs &

ref annotates pattern bindings to make them borrow rather than move. It is not a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.
By default, match statements consume all they can, which can sometimes be a problem, when you don’t really need the value to be moved and owned:

\n
1
2
3
4
5
6
7
8
let maybe_name = Some(String::from("Alice"));
// Using `ref`, the value is borrowed, not moved ...
match maybe_name {
    Some(ref n) => println!("Hello, {n}"),
    _ => println!("Hello, world"),
}
// ... so it's available here!
println!("Hello again, {}", maybe_name.unwrap_or("world".into()));
\n\n
    \n
  • & denotes that your pattern expects a reference to an object. Hence & is a part of said pattern: &Foo matches different objects than Foo does.
    • \n
  • ref indicates that you want a reference to an unpacked value. It is not matched against: Foo(ref foo) matches the same objects as Foo(foo).
    • \n
\n

Clone vs Copy

Copy 的含义

Copy 的全名是 std::marker::Copy。std::marker 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 Copy、Send、Sized、Sync。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。

\n

如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。

\n

一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。

\n

Copy 的实现条件

并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。

\n

常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。

\n

我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。

\n

Clone 的含义

Clone 的全名是 std::clone::Clone。它的完整声明是这样的：

\n
1
2
3
4
5
6
pub trait Clone : Sized {
    fn clone(&self) -> Self;
    fn clone_from(&mut self, source: &Self) {
        *self = source.clone()
    }
}
\n

它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。

\n

clone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。

\n

虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。

\n

自动 derive

绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：

\n
1
2
#[derive(Copy, Clone)]
struct MyStruct(i32);
\n

这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。

\n

通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。

\n

目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。

\n

Cell vs RefCell

    \n
  • Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(“Hello”)).get()会报错
    • \n
  • Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic
    • \n
  • 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大
    • \n
\n

AsRef vs Borrow

[WIP]

\n"},{"title":"rust tools","date":"2022-11-20T09:14:05.000Z","_content":"\n## tools\n- cargo-edit\nThis tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line\n\n- cargo whatfeatures ${crate}\n eg: `cargo whatfeatures hyper`\n","source":"_posts/rust/rust-tools.md","raw":"---\ntitle: rust tools\ndate: 2022-11-20 17:14:05\ntags: [rust]\n---\n\n## tools\n- cargo-edit\nThis tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line\n\n- cargo whatfeatures ${crate}\n eg: `cargo whatfeatures hyper`\n","slug":"rust/rust-tools","published":1,"updated":"2023-05-03T09:25:04.042Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpv00196hsjcp72dtgr","content":"

tools

    \n

  • cargo-edit
    This tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line

    \n
    • \n

  • cargo whatfeatures ${crate}
    eg: cargo whatfeatures hyper

    \n
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

tools

    \n

  • cargo-edit
    This tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line

    \n
    • \n

  • cargo whatfeatures ${crate}
    eg: cargo whatfeatures hyper

    \n
    • \n
\n"},{"title":"cryptography (1) group & field","date":"2023-06-03T06:29:26.000Z","_content":"\n\n\n\n\n## Group\na group is a set of elements \\\\( G \\\\) together with an operation \\\\( \\circ \\\\). a group has the following properties\n1. the group operation \\\\( \\circ \\\\) is closed. that is, for all \\\\( a,b, \\in G \\\\), it holds that \\\\( a \\circ b = c \\in G \\\\)\n2. the group operation is associative. That is, \\\\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\\\) for all \\\\( a,b,c \\in G \\\\)\n3. there is an element \\\\( 1 \\in G \\\\), called the neutral element (or identity element), such that \\\\( a \\circ \\ = 1 \\circ a\\\\) for all \\\\( a \\in G \\\\)\n4. For each \\\\( a \\in G \\\\) there exists an element \\\\( a^{-1} \\in G\\\\), called the inverse of a, such that \\\\( a \\circ a^{-1} = 1 \\\\).\n5. a group G is abelian (or commutative) if, furthermore, \\\\( a \\circ b = b \\circ a \\\\) for all \\\\( a, b \\in G \\\\) \n\n### Example of Group\nthe set of integers \\\\( Z_{m} = 0,1,...,m-1 \\\\) and the operation addition modulo \\\\( m \\\\) forma group with the neutral element 0. every element \\\\( a \\\\) has an inverse \\\\( -a \\\\). note that this set does not form a group with the operation multiplication gecause mose eleents \\\\( a \\\\) do not have an inverse such that \\\\( a a^{-1} = 1 \\bmod m\\\\).\n\nIn order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.\n\n## Field\nA field is a set of elements with the following properties\n- all elements of \\\\( F\\\\) form an additive group with the group operation \\\\( + \\\\) and the neutral element \\\\( 0 \\\\)\n- all element of \\\\( F \\\\) except \\\\( 0 \\\\) form a multiplicative group with the gorup operation \\\\( x \\\\) and the neutral element \\\\( 1 \\\\) .\n- when the two group operations are mixed, the distributivety law holds, i.e., for all \\\\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\\\) \n\n### Example of Field\nthe set \\\\( R \\\\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\\\( a \\\\) has an additive inverse, namely \\\\( -a \\\\), and every nonzero element \\\\( a \\\\) has a multiplicative inverse \\\\( 1 \\div a \\\\)\n\n## Galois Field\nIn cryptography, we are almose always interested in fields with a **finite** number of elements, which we call finite fields or `Galois field`. the number of elements in the field is called the `order` or `cardinality` of the field. Of fundamental importance is the following theorem\n***\na field with order `m` only exists if `m` is a prime power, i.e, \\\\( m = p^{n} \\\\), for some positive integer `n` and prime integer `p`. `p` is called the characteristic of the finite field\n***\n\nthe theorem implies that there are, for instance, finite fields with 11 elements (\\\\( 11^{1} \\\\)), or with 81 elements \\\\( 81 = 3^{4} \\\\) or with 256 elements (\\\\( 256 = 2^{8} \\\\)). \n\n## Prime Field\nthe most intuitive examples of finite fields are fields of prime order, i.e., fields with \\\\( n =1 \\\\). elements of the field \\\\( GF(p) \\\\) can be represented by integers \\\\( 0,1,..., p-1 \\\\).\n\n## Extension Field `GF(2^m)`\nIn AES the finite field contains 256 elements and is denoted as \\\\( GF(2^8) \\\\).\nHowever, if the order of a finite field is not prime, and \\\\( 2^8 \\\\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\\\( 2^8 \\\\). such fields with \\\\( m >1 \\\\) are called `extension field`. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as `polynomials`, and computation in the extension field is achieved by performing a certain type of `polynomial arithmetic`.\n\nIn the field \\\\( GF(2^8) \\\\), which is used in AES, each element \\\\( A \\in GF(2^8) \\\\) is thus represented as: \n\\\\[ A(x) = a_{7}x^7 + ...+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\\\]\nIt is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector\n\\\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\\\]\n\n### addition and subtraction in `GF(2^m)`\naddition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.\n\\\\[ A(x) = x^7 + x^6 + x^4 + 1 \\\\]\n\\\\[ B(x) = x^4 + x^2+ 1 \\\\]\n\\\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\\\]\n\n### multiplication in `GF(2^m)`\nfirstly, two elements (represented by their polynomials) of a finite field `GF(2^m)` are multiplied usig the standard polynomial multiplication rule \\\\[ A(x) \\dot B(x) = C(x) \\\\]. In general, the product polynomial \\\\[ C(x) \\\\] will have a degree higher than `m-1` and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need **irreducible** polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.\nThus, every field `GF(2^m)` requires an irreducible polynomial `P(x)` of degree `m`. \nFor AES, the irreducible polynomial is \n\\\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\\\]\nthe main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.","source":"_posts/cryptography/cryptography-01-primitive-group-and-field.md","raw":"---\ntitle: cryptography (1) group & field\ndate: 2023-06-03 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n\n## Group\na group is a set of elements \\\\( G \\\\) together with an operation \\\\( \\circ \\\\). a group has the following properties\n1. the group operation \\\\( \\circ \\\\) is closed. that is, for all \\\\( a,b, \\in G \\\\), it holds that \\\\( a \\circ b = c \\in G \\\\)\n2. the group operation is associative. That is, \\\\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\\\) for all \\\\( a,b,c \\in G \\\\)\n3. there is an element \\\\( 1 \\in G \\\\), called the neutral element (or identity element), such that \\\\( a \\circ \\ = 1 \\circ a\\\\) for all \\\\( a \\in G \\\\)\n4. For each \\\\( a \\in G \\\\) there exists an element \\\\( a^{-1} \\in G\\\\), called the inverse of a, such that \\\\( a \\circ a^{-1} = 1 \\\\).\n5. a group G is abelian (or commutative) if, furthermore, \\\\( a \\circ b = b \\circ a \\\\) for all \\\\( a, b \\in G \\\\) \n\n### Example of Group\nthe set of integers \\\\( Z_{m} = 0,1,...,m-1 \\\\) and the operation addition modulo \\\\( m \\\\) forma group with the neutral element 0. every element \\\\( a \\\\) has an inverse \\\\( -a \\\\). note that this set does not form a group with the operation multiplication gecause mose eleents \\\\( a \\\\) do not have an inverse such that \\\\( a a^{-1} = 1 \\bmod m\\\\).\n\nIn order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.\n\n## Field\nA field is a set of elements with the following properties\n- all elements of \\\\( F\\\\) form an additive group with the group operation \\\\( + \\\\) and the neutral element \\\\( 0 \\\\)\n- all element of \\\\( F \\\\) except \\\\( 0 \\\\) form a multiplicative group with the gorup operation \\\\( x \\\\) and the neutral element \\\\( 1 \\\\) .\n- when the two group operations are mixed, the distributivety law holds, i.e., for all \\\\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\\\) \n\n### Example of Field\nthe set \\\\( R \\\\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\\\( a \\\\) has an additive inverse, namely \\\\( -a \\\\), and every nonzero element \\\\( a \\\\) has a multiplicative inverse \\\\( 1 \\div a \\\\)\n\n## Galois Field\nIn cryptography, we are almose always interested in fields with a **finite** number of elements, which we call finite fields or `Galois field`. the number of elements in the field is called the `order` or `cardinality` of the field. Of fundamental importance is the following theorem\n***\na field with order `m` only exists if `m` is a prime power, i.e, \\\\( m = p^{n} \\\\), for some positive integer `n` and prime integer `p`. `p` is called the characteristic of the finite field\n***\n\nthe theorem implies that there are, for instance, finite fields with 11 elements (\\\\( 11^{1} \\\\)), or with 81 elements \\\\( 81 = 3^{4} \\\\) or with 256 elements (\\\\( 256 = 2^{8} \\\\)). \n\n## Prime Field\nthe most intuitive examples of finite fields are fields of prime order, i.e., fields with \\\\( n =1 \\\\). elements of the field \\\\( GF(p) \\\\) can be represented by integers \\\\( 0,1,..., p-1 \\\\).\n\n## Extension Field `GF(2^m)`\nIn AES the finite field contains 256 elements and is denoted as \\\\( GF(2^8) \\\\).\nHowever, if the order of a finite field is not prime, and \\\\( 2^8 \\\\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\\\( 2^8 \\\\). such fields with \\\\( m >1 \\\\) are called `extension field`. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as `polynomials`, and computation in the extension field is achieved by performing a certain type of `polynomial arithmetic`.\n\nIn the field \\\\( GF(2^8) \\\\), which is used in AES, each element \\\\( A \\in GF(2^8) \\\\) is thus represented as: \n\\\\[ A(x) = a_{7}x^7 + ...+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\\\]\nIt is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector\n\\\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\\\]\n\n### addition and subtraction in `GF(2^m)`\naddition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.\n\\\\[ A(x) = x^7 + x^6 + x^4 + 1 \\\\]\n\\\\[ B(x) = x^4 + x^2+ 1 \\\\]\n\\\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\\\]\n\n### multiplication in `GF(2^m)`\nfirstly, two elements (represented by their polynomials) of a finite field `GF(2^m)` are multiplied usig the standard polynomial multiplication rule \\\\[ A(x) \\dot B(x) = C(x) \\\\]. In general, the product polynomial \\\\[ C(x) \\\\] will have a degree higher than `m-1` and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need **irreducible** polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.\nThus, every field `GF(2^m)` requires an irreducible polynomial `P(x)` of degree `m`. \nFor AES, the irreducible polynomial is \n\\\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\\\]\nthe main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.","slug":"cryptography/cryptography-01-primitive-group-and-field","published":1,"updated":"2023-07-13T16:01:11.422Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpv001c6hsjfftj9tr6","content":"\n\n\n\n\n

Group

a group is a set of elements \\( G \\) together with an operation \\( \\circ \\). a group has the following properties

\n
    \n
  1. the group operation \\( \\circ \\) is closed. that is, for all \\( a,b, \\in G \\), it holds that \\( a \\circ b = c \\in G \\)
    2. \n
  2. the group operation is associative. That is, \\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\) for all \\( a,b,c \\in G \\)
    3. \n
  3. there is an element \\( 1 \\in G \\), called the neutral element (or identity element), such that \\( a \\circ \\ = 1 \\circ a\\) for all \\( a \\in G \\)
    4. \n
  4. For each \\( a \\in G \\) there exists an element \\( a^{-1} \\in G\\), called the inverse of a, such that \\( a \\circ a^{-1} = 1 \\).
    5. \n
  5. a group G is abelian (or commutative) if, furthermore, \\( a \\circ b = b \\circ a \\) for all \\( a, b \\in G \\)
    6. \n
\n

Example of Group

the set of integers \\( Z_{m} = 0,1,…,m-1 \\) and the operation addition modulo \\( m \\) forma group with the neutral element 0. every element \\( a \\) has an inverse \\( -a \\). note that this set does not form a group with the operation multiplication gecause mose eleents \\( a \\) do not have an inverse such that \\( a a^{-1} = 1 \\bmod m\\).

\n

In order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.

\n

Field

A field is a set of elements with the following properties

\n
    \n
  • all elements of \\( F\\) form an additive group with the group operation \\( + \\) and the neutral element \\( 0 \\)
    • \n
  • all element of \\( F \\) except \\( 0 \\) form a multiplicative group with the gorup operation \\( x \\) and the neutral element \\( 1 \\) .
    • \n
  • when the two group operations are mixed, the distributivety law holds, i.e., for all \\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\)
    • \n
\n

Example of Field

the set \\( R \\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\( a \\) has an additive inverse, namely \\( -a \\), and every nonzero element \\( a \\) has a multiplicative inverse \\( 1 \\div a \\)

\n

Galois Field

In cryptography, we are almose always interested in fields with a finite number of elements, which we call finite fields or Galois field. the number of elements in the field is called the order or cardinality of the field. Of fundamental importance is the following theorem

\n
\n

a field with order m only exists if m is a prime power, i.e, \\( m = p^{n} \\), for some positive integer n and prime integer p. p is called the characteristic of the finite field

\n
\n

the theorem implies that there are, for instance, finite fields with 11 elements (\\( 11^{1} \\)), or with 81 elements \\( 81 = 3^{4} \\) or with 256 elements (\\( 256 = 2^{8} \\)).

\n

Prime Field

the most intuitive examples of finite fields are fields of prime order, i.e., fields with \\( n =1 \\). elements of the field \\( GF(p) \\) can be represented by integers \\( 0,1,…, p-1 \\).

\n

Extension Field GF(2^m)

In AES the finite field contains 256 elements and is denoted as \\( GF(2^8) \\).
However, if the order of a finite field is not prime, and \\( 2^8 \\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\( 2^8 \\). such fields with \\( m >1 \\) are called extension field. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as polynomials, and computation in the extension field is achieved by performing a certain type of polynomial arithmetic.

\n

In the field \\( GF(2^8) \\), which is used in AES, each element \\( A \\in GF(2^8) \\) is thus represented as:
\\[ A(x) = a_{7}x^7 + …+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\]
It is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector
\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\]

\n

addition and subtraction in GF(2^m)

addition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.
\\[ A(x) = x^7 + x^6 + x^4 + 1 \\]
\\[ B(x) = x^4 + x^2+ 1 \\]
\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\]

\n

multiplication in GF(2^m)

firstly, two elements (represented by their polynomials) of a finite field GF(2^m) are multiplied usig the standard polynomial multiplication rule \\[ A(x) \\dot B(x) = C(x) \\]. In general, the product polynomial \\[ C(x) \\] will have a degree higher than m-1 and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need irreducible polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.
Thus, every field GF(2^m) requires an irreducible polynomial P(x) of degree m.
For AES, the irreducible polynomial is
\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\]
the main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n\n

Group

a group is a set of elements \\( G \\) together with an operation \\( \\circ \\). a group has the following properties

\n
    \n
  1. the group operation \\( \\circ \\) is closed. that is, for all \\( a,b, \\in G \\), it holds that \\( a \\circ b = c \\in G \\)
    2. \n
  2. the group operation is associative. That is, \\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\) for all \\( a,b,c \\in G \\)
    3. \n
  3. there is an element \\( 1 \\in G \\), called the neutral element (or identity element), such that \\( a \\circ \\ = 1 \\circ a\\) for all \\( a \\in G \\)
    4. \n
  4. For each \\( a \\in G \\) there exists an element \\( a^{-1} \\in G\\), called the inverse of a, such that \\( a \\circ a^{-1} = 1 \\).
    5. \n
  5. a group G is abelian (or commutative) if, furthermore, \\( a \\circ b = b \\circ a \\) for all \\( a, b \\in G \\)
    6. \n
\n

Example of Group

the set of integers \\( Z_{m} = 0,1,…,m-1 \\) and the operation addition modulo \\( m \\) forma group with the neutral element 0. every element \\( a \\) has an inverse \\( -a \\). note that this set does not form a group with the operation multiplication gecause mose eleents \\( a \\) do not have an inverse such that \\( a a^{-1} = 1 \\bmod m\\).

\n

In order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.

\n

Field

A field is a set of elements with the following properties

\n
    \n
  • all elements of \\( F\\) form an additive group with the group operation \\( + \\) and the neutral element \\( 0 \\)
    • \n
  • all element of \\( F \\) except \\( 0 \\) form a multiplicative group with the gorup operation \\( x \\) and the neutral element \\( 1 \\) .
    • \n
  • when the two group operations are mixed, the distributivety law holds, i.e., for all \\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\)
    • \n
\n

Example of Field

the set \\( R \\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\( a \\) has an additive inverse, namely \\( -a \\), and every nonzero element \\( a \\) has a multiplicative inverse \\( 1 \\div a \\)

\n

Galois Field

In cryptography, we are almose always interested in fields with a finite number of elements, which we call finite fields or Galois field. the number of elements in the field is called the order or cardinality of the field. Of fundamental importance is the following theorem

\n
\n

a field with order m only exists if m is a prime power, i.e, \\( m = p^{n} \\), for some positive integer n and prime integer p. p is called the characteristic of the finite field

\n
\n

the theorem implies that there are, for instance, finite fields with 11 elements (\\( 11^{1} \\)), or with 81 elements \\( 81 = 3^{4} \\) or with 256 elements (\\( 256 = 2^{8} \\)).

\n

Prime Field

the most intuitive examples of finite fields are fields of prime order, i.e., fields with \\( n =1 \\). elements of the field \\( GF(p) \\) can be represented by integers \\( 0,1,…, p-1 \\).

\n

Extension Field GF(2^m)

In AES the finite field contains 256 elements and is denoted as \\( GF(2^8) \\).
However, if the order of a finite field is not prime, and \\( 2^8 \\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\( 2^8 \\). such fields with \\( m >1 \\) are called extension field. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as polynomials, and computation in the extension field is achieved by performing a certain type of polynomial arithmetic.

\n

In the field \\( GF(2^8) \\), which is used in AES, each element \\( A \\in GF(2^8) \\) is thus represented as:
\\[ A(x) = a_{7}x^7 + …+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\]
It is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector
\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\]

\n

addition and subtraction in GF(2^m)

addition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.
\\[ A(x) = x^7 + x^6 + x^4 + 1 \\]
\\[ B(x) = x^4 + x^2+ 1 \\]
\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\]

\n

multiplication in GF(2^m)

firstly, two elements (represented by their polynomials) of a finite field GF(2^m) are multiplied usig the standard polynomial multiplication rule \\[ A(x) \\dot B(x) = C(x) \\]. In general, the product polynomial \\[ C(x) \\] will have a degree higher than m-1 and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need irreducible polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.
Thus, every field GF(2^m) requires an irreducible polynomial P(x) of degree m.
For AES, the irreducible polynomial is
\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\]
the main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.

\n"},{"title":"cryptography (2) elliptic curve","date":"2023-06-10T06:29:26.000Z","_content":"\n\n\n\n## elliptic curve definition\nfor cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields `GF(p)`, where all arithmetic is performed modulo a prime p.\n***\nthe elliptic curve over \\\\( Z_{p}, p>3 \\\\), is the set of all pairs \\\\( (x,y) \\in Z_{p} \\\\) which fulfill\n \\\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\\\]\ntogether with an imaginary point of infinity \\\\( \\mathcal{O} \\\\), where\n \\\\[ a,b \\in Z_{p} \\\\]\n and the condition \\\\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\\\)\n***\nthe definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\\\( -16(4a^3) + 27b^2 \\\\) is nonzero.\n\n## operations on elliptic curve\n![point addition](/images/elliptic_curve/point_addition.webp)\nlet's denote the group operation with the addition symbol `+`. \"addition\" means that given two points and their coordinates, say \\\\( P = (x_1, y_1) \\\\) and \\\\( Q = (x_2, y_2) \\\\), we have to compute the coordidnate of a third point \\\\( R (x_3, y_3) \\\\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling `(P+P = 2P)` is a tangent line through the point P, and the rest is same.\nthe fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below \n***\n \\\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\\\]\n \\\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\\\]\n where \n\\begin{equation}\ns = \n\\begin{cases}\n\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr\n\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})\n\\end{cases}\n\\end{equation}\n***\nnote that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.\nFurthmore, we define an abstract point at infinity as the neutral element \\\\( \\mathcal{O} \\\\). \n\\\\[ P + \\mathcal{O} = P\\\\]\naccording the group definition, we can now also define the inverse `-P` of any group element P as\n\\\\[ P + (-P) = \\mathcal{O}\\\\]\nTherefore, the inverse of \\\\( P = (x_p, y_p) \\\\) is just \\\\( P = (x_p, -y_p)\\\\). since \\\\( -y_p \\equiv p - y_p\\\\), hence\n\\\\[ -P = (x_p, p-y_p) \\\\]\n\n## DLP(discrete log problem) with Elliptic curve\nto set up DL cryptosystems it is important to know the order of the group.\nHasse's theorem\n***\ngiven an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:\n\\\\[ p+1-2\\sqrt{p} \\leq \\\\#E \\leq p+1+2\\sqrt{p} \\\\]\n***\nElliptic Curved Discrete Logarithm Problem (ECDLP)\n***\nGiven an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\\\( 1 \\leq d \\leq \\\\#E \\\\), such that:\n\\\\[ P + P + ....+ P = dP = T \\\\]\n***\nIn cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\\\( T = (x_T, y_T)\\\\)\nUsually a **square-and-multiply** algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post). \n","source":"_posts/cryptography/cryptography-02-elliptic-curve.md","raw":"---\ntitle: cryptography (2) elliptic curve\ndate: 2023-06-10 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n## elliptic curve definition\nfor cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields `GF(p)`, where all arithmetic is performed modulo a prime p.\n***\nthe elliptic curve over \\\\( Z_{p}, p>3 \\\\), is the set of all pairs \\\\( (x,y) \\in Z_{p} \\\\) which fulfill\n \\\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\\\]\ntogether with an imaginary point of infinity \\\\( \\mathcal{O} \\\\), where\n \\\\[ a,b \\in Z_{p} \\\\]\n and the condition \\\\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\\\)\n***\nthe definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\\\( -16(4a^3) + 27b^2 \\\\) is nonzero.\n\n## operations on elliptic curve\n![point addition](/images/elliptic_curve/point_addition.webp)\nlet's denote the group operation with the addition symbol `+`. \"addition\" means that given two points and their coordinates, say \\\\( P = (x_1, y_1) \\\\) and \\\\( Q = (x_2, y_2) \\\\), we have to compute the coordidnate of a third point \\\\( R (x_3, y_3) \\\\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling `(P+P = 2P)` is a tangent line through the point P, and the rest is same.\nthe fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below \n***\n \\\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\\\]\n \\\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\\\]\n where \n\\begin{equation}\ns = \n\\begin{cases}\n\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr\n\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})\n\\end{cases}\n\\end{equation}\n***\nnote that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.\nFurthmore, we define an abstract point at infinity as the neutral element \\\\( \\mathcal{O} \\\\). \n\\\\[ P + \\mathcal{O} = P\\\\]\naccording the group definition, we can now also define the inverse `-P` of any group element P as\n\\\\[ P + (-P) = \\mathcal{O}\\\\]\nTherefore, the inverse of \\\\( P = (x_p, y_p) \\\\) is just \\\\( P = (x_p, -y_p)\\\\). since \\\\( -y_p \\equiv p - y_p\\\\), hence\n\\\\[ -P = (x_p, p-y_p) \\\\]\n\n## DLP(discrete log problem) with Elliptic curve\nto set up DL cryptosystems it is important to know the order of the group.\nHasse's theorem\n***\ngiven an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:\n\\\\[ p+1-2\\sqrt{p} \\leq \\\\#E \\leq p+1+2\\sqrt{p} \\\\]\n***\nElliptic Curved Discrete Logarithm Problem (ECDLP)\n***\nGiven an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\\\( 1 \\leq d \\leq \\\\#E \\\\), such that:\n\\\\[ P + P + ....+ P = dP = T \\\\]\n***\nIn cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\\\( T = (x_T, y_T)\\\\)\nUsually a **square-and-multiply** algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post). \n","slug":"cryptography/cryptography-02-elliptic-curve","published":1,"updated":"2023-07-13T16:01:15.193Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpw001e6hsj1lfmhvaa","content":"\n\n\n\n

elliptic curve definition

for cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields GF(p), where all arithmetic is performed modulo a prime p.

\n
\n

the elliptic curve over \\( Z_{p}, p>3 \\), is the set of all pairs \\( (x,y) \\in Z_{p} \\) which fulfill
\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\]
together with an imaginary point of infinity \\( \\mathcal{O} \\), where
\\[ a,b \\in Z_{p} \\]
and the condition \\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\)

\n
\n

the definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\( -16(4a^3) + 27b^2 \\) is nonzero.

\n

operations on elliptic curve

\"point
let’s denote the group operation with the addition symbol +. “addition” means that given two points and their coordinates, say \\( P = (x_1, y_1) \\) and \\( Q = (x_2, y_2) \\), we have to compute the coordidnate of a third point \\( R (x_3, y_3) \\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling (P+P = 2P) is a tangent line through the point P, and the rest is same.
the fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below

\n
\n

\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\]
\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\]
where
\\begin{equation}
s =
\\begin{cases}
\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr
\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})
\\end{cases}
\\end{equation}

\n
\n

note that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.
Furthmore, we define an abstract point at infinity as the neutral element \\( \\mathcal{O} \\).
\\[ P + \\mathcal{O} = P\\]
according the group definition, we can now also define the inverse -P of any group element P as
\\[ P + (-P) = \\mathcal{O}\\]
Therefore, the inverse of \\( P = (x_p, y_p) \\) is just \\( P = (x_p, -y_p)\\). since \\( -y_p \\equiv p - y_p\\), hence
\\[ -P = (x_p, p-y_p) \\]

\n

DLP(discrete log problem) with Elliptic curve

to set up DL cryptosystems it is important to know the order of the group.
Hasse’s theorem

\n
\n

given an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:
\\[ p+1-2\\sqrt{p} \\leq \\#E \\leq p+1+2\\sqrt{p} \\]

\n
\n

Elliptic Curved Discrete Logarithm Problem (ECDLP)

\n
\n

Given an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\( 1 \\leq d \\leq \\#E \\), such that:
\\[ P + P + ….+ P = dP = T \\]

\n
\n

In cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\( T = (x_T, y_T)\\)
Usually a square-and-multiply algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post).

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

elliptic curve definition

for cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields GF(p), where all arithmetic is performed modulo a prime p.

\n
\n

the elliptic curve over \\( Z_{p}, p>3 \\), is the set of all pairs \\( (x,y) \\in Z_{p} \\) which fulfill
\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\]
together with an imaginary point of infinity \\( \\mathcal{O} \\), where
\\[ a,b \\in Z_{p} \\]
and the condition \\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\)

\n
\n

the definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\( -16(4a^3) + 27b^2 \\) is nonzero.

\n

operations on elliptic curve

\"point
let’s denote the group operation with the addition symbol +. “addition” means that given two points and their coordinates, say \\( P = (x_1, y_1) \\) and \\( Q = (x_2, y_2) \\), we have to compute the coordidnate of a third point \\( R (x_3, y_3) \\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling (P+P = 2P) is a tangent line through the point P, and the rest is same.
the fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below

\n
\n

\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\]
\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\]
where
\\begin{equation}
s =
\\begin{cases}
\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr
\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})
\\end{cases}
\\end{equation}

\n
\n

note that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.
Furthmore, we define an abstract point at infinity as the neutral element \\( \\mathcal{O} \\).
\\[ P + \\mathcal{O} = P\\]
according the group definition, we can now also define the inverse -P of any group element P as
\\[ P + (-P) = \\mathcal{O}\\]
Therefore, the inverse of \\( P = (x_p, y_p) \\) is just \\( P = (x_p, -y_p)\\). since \\( -y_p \\equiv p - y_p\\), hence
\\[ -P = (x_p, p-y_p) \\]

\n

DLP(discrete log problem) with Elliptic curve

to set up DL cryptosystems it is important to know the order of the group.
Hasse’s theorem

\n
\n

given an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:
\\[ p+1-2\\sqrt{p} \\leq \\#E \\leq p+1+2\\sqrt{p} \\]

\n
\n

Elliptic Curved Discrete Logarithm Problem (ECDLP)

\n
\n

Given an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\( 1 \\leq d \\leq \\#E \\), such that:
\\[ P + P + ….+ P = dP = T \\]

\n
\n

In cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\( T = (x_T, y_T)\\)
Usually a square-and-multiply algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post).

\n"},{"title":"paillier encryption","date":"2023-02-23T13:25:41.000Z","_content":"\n\n\n\n## fundamentals\n1. fundamental theorem of arighmetic\nthe fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors [wiki](https://en.wikipedia.org/wiki/Fundamental_theorem_of_arithmetic)\n2. Euler's totient function\nIn number theory, Euler's totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\\\( \\phi (n) \\\\), and may also be called Euler's phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\\\( Z_{n}^{\\ast } \\\\), and \\\\[ \\phi(n) = |Z_n^{\\ast }| \\\\]\n3. if p is prime, then \\\\( Z_p^{\\ast } = Z_p \\\\), \\\\( \\phi(p) = p-1 \\\\)\n4. if p is prime, for any integer r, then \\\\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\\\)\n5. Euler's totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\\\(\\phi(mn) = \\phi(m)\\phi(n)\\\\)\n6. Euler's product formula, it states\n\\\\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\\\]\nwhere the product is over the distinct prime numbers dividing n.\n7. Euler's theorem\nif \\\\(a\\\\) and \\\\(n\\\\) are coprime positive integers, and \\\\( \\phi(n)\\\\) is Euler's totient function, then \\\\(a\\\\) raised to the power \\\\(\\phi(n)\\\\) is congruent to 1 modulo n; that is\n\\\\[a^{\\phi(n)} \\equiv 1 \\bmod n\\\\]\n8. according to 7, we have \\\\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\\\). then\n\\\\[ a^{-1} = a^{\\phi(n)-1} \\\\]\n9. Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\n10. Binomial theorem\nit states\n\\\\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + ...\\\\]\nobserve that, the higher degree could be divided by \\\\(n^2\\\\). we have\n\\\\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\\\]\ntherefore, \\\\( y - 1 \\equiv nx \\bmod n^2 \\\\). then we have\n\\\\[ x \\equiv \\frac{y-1}{n} \\bmod n \\\\].\nIn paillier, later we define \\\\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\\\)\ntherefore\n\\\\[ L(y \\bmod n^2) \\equiv x \\bmod n \\\\]\n\n\n## Paillier\n1. key generation\n`KeyGen() -> (pk, sk)`\nrandomly select two big prime numbers \\\\(p, q\\\\). it shoud satisfy \\\\(gcd(pq, (p-1)(q-1)) =1 \\\\), \\\\(p\\\\) and \\\\(q\\\\) should have similar bit length. let \\\\( n = pq \\\\), \\\\(\\lambda = lcm(p-1, q-1)\\\\). randomly sample \\\\( g \\in Z_{n^2}^{\\ast}\\\\). to simplify, let \\\\( g = n+1\\\\). we have\n\\\\[ pk=(n,g) \\\\]\n\\\\[ sk = (\\lambda)\\\\]\n\n2. encryption\n`Enc(pk, m) -> c`\nrandomly sample \\\\( r \\in Z_{n}^{\\ast}\\\\), then also have \\\\( r \\in Z_{n^2}^{\\ast}\\\\), cypher is calculated\n\\\\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\\\]\n\n3. Decryption\n`Dec(sk, c) -> m`\nLet \\\\(L(x) = \\frac{x-1}{n} \\\\), we have message\n\\\\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\\\]\n\n4. proof of correctness\nbased on Eq(1), we have \\\\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\\\]\nwhere \\\\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\\\), which is proved by Carmichael theorem later on. then Eq(3) becomes\n \\\\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nsince \\\\( g = n+1\\\\), we have\n\\\\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nAccording to Eq(0.2), we have\n\\\\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\\\]\n\\\\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\\\]\ntherefore, based on definition given by Eq(0.3) we have\n\\\\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\\\]\nSubstitute Eq(1.6) into Eq(1.8), we have\n\\\\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\\\]\nFurther, we have\n\\\\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\\\]\nSub Eq(1.7) into Eq(1.10), we have\n\\\\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\\\]\nAt last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)\n\\\\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\\\]\nproved!!!\n\n5. Carmichael theorem\nIn number theory, a branch of mathematics, the Carmichael function \\\\(λ(n)\\\\) of a positive integer n is the smallest positive integer m such that \\\\(a^{m}\\equiv 1{\\pmod {n}}\\\\) (similar but different from Euler's totient function). Carmichael's λ function, the reduced totient function, and the least universal exponent function\n![carmichael theorem](/images/paillier/carmichael_thorem.png)\n![](/images/paillier/carmichael_thorem_2.png)\nlet \\\\( n = pq\\\\), where p and q are prime numbers; \\\\( \\phi(n)\\\\) is the Euler's totient function. Let \\\\(\\lambda(n)\\\\) denotes carmichael function. We have \\\\(\\phi(n)=(p-1)(q-1)\\\\) and \\\\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\\\).\n\nSince \\\\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\\\) (according to Eq(0.1)). Thereby, for any \\\\( w \\in Z_{n^2}^{\\ast}\\\\)\n\\\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\\\]\n\n\\\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\\\]\nEq(1.13) is just Carmichael's function\n\nBased on Carmichael's theorem\n\\\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\\\] \ntherefore, we have\n\n\\\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\\\]\n\n6. Addition homomorphic\n![homomorphic addition](/images/paillier/homomorphic_addition.png)\n\n7. Multiplication homomorphic \n![homomorphic multiplication](/images/paillier/homomorphic_mul.png)\n## references\n- [csdn post](https://blog.csdn.net/qq_42328228/article/details/109349590)","source":"_posts/cryptography/paillier-encryption.md","raw":"---\ntitle: paillier encryption\ndate: 2023-02-23 21:25:41\ntags: [cryptography]\n---\n\n\n\n\n## fundamentals\n1. fundamental theorem of arighmetic\nthe fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors [wiki](https://en.wikipedia.org/wiki/Fundamental_theorem_of_arithmetic)\n2. Euler's totient function\nIn number theory, Euler's totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\\\( \\phi (n) \\\\), and may also be called Euler's phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\\\( Z_{n}^{\\ast } \\\\), and \\\\[ \\phi(n) = |Z_n^{\\ast }| \\\\]\n3. if p is prime, then \\\\( Z_p^{\\ast } = Z_p \\\\), \\\\( \\phi(p) = p-1 \\\\)\n4. if p is prime, for any integer r, then \\\\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\\\)\n5. Euler's totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\\\(\\phi(mn) = \\phi(m)\\phi(n)\\\\)\n6. Euler's product formula, it states\n\\\\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\\\]\nwhere the product is over the distinct prime numbers dividing n.\n7. Euler's theorem\nif \\\\(a\\\\) and \\\\(n\\\\) are coprime positive integers, and \\\\( \\phi(n)\\\\) is Euler's totient function, then \\\\(a\\\\) raised to the power \\\\(\\phi(n)\\\\) is congruent to 1 modulo n; that is\n\\\\[a^{\\phi(n)} \\equiv 1 \\bmod n\\\\]\n8. according to 7, we have \\\\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\\\). then\n\\\\[ a^{-1} = a^{\\phi(n)-1} \\\\]\n9. Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\n10. Binomial theorem\nit states\n\\\\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + ...\\\\]\nobserve that, the higher degree could be divided by \\\\(n^2\\\\). we have\n\\\\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\\\]\ntherefore, \\\\( y - 1 \\equiv nx \\bmod n^2 \\\\). then we have\n\\\\[ x \\equiv \\frac{y-1}{n} \\bmod n \\\\].\nIn paillier, later we define \\\\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\\\)\ntherefore\n\\\\[ L(y \\bmod n^2) \\equiv x \\bmod n \\\\]\n\n\n## Paillier\n1. key generation\n`KeyGen() -> (pk, sk)`\nrandomly select two big prime numbers \\\\(p, q\\\\). it shoud satisfy \\\\(gcd(pq, (p-1)(q-1)) =1 \\\\), \\\\(p\\\\) and \\\\(q\\\\) should have similar bit length. let \\\\( n = pq \\\\), \\\\(\\lambda = lcm(p-1, q-1)\\\\). randomly sample \\\\( g \\in Z_{n^2}^{\\ast}\\\\). to simplify, let \\\\( g = n+1\\\\). we have\n\\\\[ pk=(n,g) \\\\]\n\\\\[ sk = (\\lambda)\\\\]\n\n2. encryption\n`Enc(pk, m) -> c`\nrandomly sample \\\\( r \\in Z_{n}^{\\ast}\\\\), then also have \\\\( r \\in Z_{n^2}^{\\ast}\\\\), cypher is calculated\n\\\\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\\\]\n\n3. Decryption\n`Dec(sk, c) -> m`\nLet \\\\(L(x) = \\frac{x-1}{n} \\\\), we have message\n\\\\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\\\]\n\n4. proof of correctness\nbased on Eq(1), we have \\\\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\\\]\nwhere \\\\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\\\), which is proved by Carmichael theorem later on. then Eq(3) becomes\n \\\\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nsince \\\\( g = n+1\\\\), we have\n\\\\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nAccording to Eq(0.2), we have\n\\\\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\\\]\n\\\\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\\\]\ntherefore, based on definition given by Eq(0.3) we have\n\\\\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\\\]\nSubstitute Eq(1.6) into Eq(1.8), we have\n\\\\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\\\]\nFurther, we have\n\\\\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\\\]\nSub Eq(1.7) into Eq(1.10), we have\n\\\\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\\\]\nAt last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)\n\\\\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\\\]\nproved!!!\n\n5. Carmichael theorem\nIn number theory, a branch of mathematics, the Carmichael function \\\\(λ(n)\\\\) of a positive integer n is the smallest positive integer m such that \\\\(a^{m}\\equiv 1{\\pmod {n}}\\\\) (similar but different from Euler's totient function). Carmichael's λ function, the reduced totient function, and the least universal exponent function\n![carmichael theorem](/images/paillier/carmichael_thorem.png)\n![](/images/paillier/carmichael_thorem_2.png)\nlet \\\\( n = pq\\\\), where p and q are prime numbers; \\\\( \\phi(n)\\\\) is the Euler's totient function. Let \\\\(\\lambda(n)\\\\) denotes carmichael function. We have \\\\(\\phi(n)=(p-1)(q-1)\\\\) and \\\\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\\\).\n\nSince \\\\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\\\) (according to Eq(0.1)). Thereby, for any \\\\( w \\in Z_{n^2}^{\\ast}\\\\)\n\\\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\\\]\n\n\\\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\\\]\nEq(1.13) is just Carmichael's function\n\nBased on Carmichael's theorem\n\\\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\\\] \ntherefore, we have\n\n\\\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\\\]\n\n6. Addition homomorphic\n![homomorphic addition](/images/paillier/homomorphic_addition.png)\n\n7. Multiplication homomorphic \n![homomorphic multiplication](/images/paillier/homomorphic_mul.png)\n## references\n- [csdn post](https://blog.csdn.net/qq_42328228/article/details/109349590)","slug":"cryptography/paillier-encryption","published":1,"updated":"2023-04-24T06:31:44.177Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpw001h6hsj7tdyclkv","content":"\n\n\n

fundamentals

    \n
  1. fundamental theorem of arighmetic
    the fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors wiki
    2. \n
  2. Euler’s totient function
    In number theory, Euler’s totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\( \\phi (n) \\), and may also be called Euler’s phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\( Z_{n}^{\\ast } \\), and \\[ \\phi(n) = |Z_n^{\\ast }| \\]
    3. \n
  3. if p is prime, then \\( Z_p^{\\ast } = Z_p \\), \\( \\phi(p) = p-1 \\)
    4. \n
  4. if p is prime, for any integer r, then \\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\)
    5. \n
  5. Euler’s totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\(\\phi(mn) = \\phi(m)\\phi(n)\\)
    6. \n
  6. Euler’s product formula, it states
    \\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\]
    where the product is over the distinct prime numbers dividing n.
    7. \n
  7. Euler’s theorem
    if \\(a\\) and \\(n\\) are coprime positive integers, and \\( \\phi(n)\\) is Euler’s totient function, then \\(a\\) raised to the power \\(\\phi(n)\\) is congruent to 1 modulo n; that is
    \\[a^{\\phi(n)} \\equiv 1 \\bmod n\\]
    8. \n
  8. according to 7, we have \\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\). then
    \\[ a^{-1} = a^{\\phi(n)-1} \\]
    9. \n
  9. Fermat’s little theorem
    Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
    \\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
    \\[ a^{p} \\equiv a \\bmod p\\]
    10. \n
  10. Binomial theorem
    it states
    \\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + …\\]
    observe that, the higher degree could be divided by \\(n^2\\). we have
    \\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\]
    therefore, \\( y - 1 \\equiv nx \\bmod n^2 \\). then we have
    \\[ x \\equiv \\frac{y-1}{n} \\bmod n \\].
    In paillier, later we define \\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\)
    therefore
    \\[ L(y \\bmod n^2) \\equiv x \\bmod n \\]
    11. \n
\n

Paillier

    \n

  1. key generation
    KeyGen() -> (pk, sk)
    randomly select two big prime numbers \\(p, q\\). it shoud satisfy \\(gcd(pq, (p-1)(q-1)) =1 \\), \\(p\\) and \\(q\\) should have similar bit length. let \\( n = pq \\), \\(\\lambda = lcm(p-1, q-1)\\). randomly sample \\( g \\in Z_{n^2}^{\\ast}\\). to simplify, let \\( g = n+1\\). we have
    \\[ pk=(n,g) \\]
    \\[ sk = (\\lambda)\\]

    \n
    2. \n

  2. encryption
    Enc(pk, m) -> c
    randomly sample \\( r \\in Z_{n}^{\\ast}\\), then also have \\( r \\in Z_{n^2}^{\\ast}\\), cypher is calculated
    \\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\]

    \n
    3. \n

  3. Decryption
    Dec(sk, c) -> m
    Let \\(L(x) = \\frac{x-1}{n} \\), we have message
    \\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\]

    \n
    4. \n

  4. proof of correctness
    based on Eq(1), we have \\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\]
    where \\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\), which is proved by Carmichael theorem later on. then Eq(3) becomes
    \\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\]
    since \\( g = n+1\\), we have
    \\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\]
    According to Eq(0.2), we have
    \\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\]
    \\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\]
    therefore, based on definition given by Eq(0.3) we have
    \\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\]
    Substitute Eq(1.6) into Eq(1.8), we have
    \\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\]
    Further, we have
    \\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\]
    Sub Eq(1.7) into Eq(1.10), we have
    \\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\]
    At last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)
    \\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\]
    proved!!!

    \n
    5. \n

  5. Carmichael theorem
    In number theory, a branch of mathematics, the Carmichael function \\(λ(n)\\) of a positive integer n is the smallest positive integer m such that \\(a^{m}\\equiv 1{\\pmod {n}}\\) (similar but different from Euler’s totient function). Carmichael’s λ function, the reduced totient function, and the least universal exponent function
    \"carmichael

    let \\( n = pq\\), where p and q are prime numbers; \\( \\phi(n)\\) is the Euler’s totient function. Let \\(\\lambda(n)\\) denotes carmichael function. We have \\(\\phi(n)=(p-1)(q-1)\\) and \\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\).

    \n
    6. \n
\n

Since \\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\) (according to Eq(0.1)). Thereby, for any \\( w \\in Z_{n^2}^{\\ast}\\)
\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\]

\n

\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\]
Eq(1.13) is just Carmichael’s function

\n

Based on Carmichael’s theorem
\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\]
therefore, we have

\n

\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\]

\n
    \n

  1. Addition homomorphic
    \"homomorphic

    \n
    2. \n

  2. Multiplication homomorphic
    \"homomorphic

    \n
    3. \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

fundamentals

    \n
  1. fundamental theorem of arighmetic
    the fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors wiki
    2. \n
  2. Euler’s totient function
    In number theory, Euler’s totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\( \\phi (n) \\), and may also be called Euler’s phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\( Z_{n}^{\\ast } \\), and \\[ \\phi(n) = |Z_n^{\\ast }| \\]
    3. \n
  3. if p is prime, then \\( Z_p^{\\ast } = Z_p \\), \\( \\phi(p) = p-1 \\)
    4. \n
  4. if p is prime, for any integer r, then \\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\)
    5. \n
  5. Euler’s totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\(\\phi(mn) = \\phi(m)\\phi(n)\\)
    6. \n
  6. Euler’s product formula, it states
    \\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\]
    where the product is over the distinct prime numbers dividing n.
    7. \n
  7. Euler’s theorem
    if \\(a\\) and \\(n\\) are coprime positive integers, and \\( \\phi(n)\\) is Euler’s totient function, then \\(a\\) raised to the power \\(\\phi(n)\\) is congruent to 1 modulo n; that is
    \\[a^{\\phi(n)} \\equiv 1 \\bmod n\\]
    8. \n
  8. according to 7, we have \\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\). then
    \\[ a^{-1} = a^{\\phi(n)-1} \\]
    9. \n
  9. Fermat’s little theorem
    Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
    \\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
    \\[ a^{p} \\equiv a \\bmod p\\]
    10. \n
  10. Binomial theorem
    it states
    \\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + …\\]
    observe that, the higher degree could be divided by \\(n^2\\). we have
    \\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\]
    therefore, \\( y - 1 \\equiv nx \\bmod n^2 \\). then we have
    \\[ x \\equiv \\frac{y-1}{n} \\bmod n \\].
    In paillier, later we define \\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\)
    therefore
    \\[ L(y \\bmod n^2) \\equiv x \\bmod n \\]
    11. \n
\n

Paillier

    \n

  1. key generation
    KeyGen() -> (pk, sk)
    randomly select two big prime numbers \\(p, q\\). it shoud satisfy \\(gcd(pq, (p-1)(q-1)) =1 \\), \\(p\\) and \\(q\\) should have similar bit length. let \\( n = pq \\), \\(\\lambda = lcm(p-1, q-1)\\). randomly sample \\( g \\in Z_{n^2}^{\\ast}\\). to simplify, let \\( g = n+1\\). we have
    \\[ pk=(n,g) \\]
    \\[ sk = (\\lambda)\\]

    \n
    2. \n

  2. encryption
    Enc(pk, m) -> c
    randomly sample \\( r \\in Z_{n}^{\\ast}\\), then also have \\( r \\in Z_{n^2}^{\\ast}\\), cypher is calculated
    \\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\]

    \n
    3. \n

  3. Decryption
    Dec(sk, c) -> m
    Let \\(L(x) = \\frac{x-1}{n} \\), we have message
    \\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\]

    \n
    4. \n

  4. proof of correctness
    based on Eq(1), we have \\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\]
    where \\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\), which is proved by Carmichael theorem later on. then Eq(3) becomes
    \\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\]
    since \\( g = n+1\\), we have
    \\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\]
    According to Eq(0.2), we have
    \\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\]
    \\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\]
    therefore, based on definition given by Eq(0.3) we have
    \\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\]
    Substitute Eq(1.6) into Eq(1.8), we have
    \\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\]
    Further, we have
    \\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\]
    Sub Eq(1.7) into Eq(1.10), we have
    \\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\]
    At last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)
    \\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\]
    proved!!!

    \n
    5. \n

  5. Carmichael theorem
    In number theory, a branch of mathematics, the Carmichael function \\(λ(n)\\) of a positive integer n is the smallest positive integer m such that \\(a^{m}\\equiv 1{\\pmod {n}}\\) (similar but different from Euler’s totient function). Carmichael’s λ function, the reduced totient function, and the least universal exponent function
    \"carmichael

    let \\( n = pq\\), where p and q are prime numbers; \\( \\phi(n)\\) is the Euler’s totient function. Let \\(\\lambda(n)\\) denotes carmichael function. We have \\(\\phi(n)=(p-1)(q-1)\\) and \\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\).

    \n
    6. \n
\n

Since \\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\) (according to Eq(0.1)). Thereby, for any \\( w \\in Z_{n^2}^{\\ast}\\)
\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\]

\n

\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\]
Eq(1.13) is just Carmichael’s function

\n

Based on Carmichael’s theorem
\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\]
therefore, we have

\n

\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\]

\n
    \n

  1. Addition homomorphic
    \"homomorphic

    \n
    2. \n

  2. Multiplication homomorphic
    \"homomorphic

    \n
    3. \n
\n

references

\n"},{"title":"cryptography (3) RSA cryptosystem","date":"2023-06-17T06:29:26.000Z","_content":"\n\n\n\n## introduction\nthe security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].\n\n## Euclidean Algorithm\ntodo!()\n## Extended Euclidean Algorithm\ntodo!()\n\n## Euler's Phi Function\nwe consider the ring \\\\( Z_m\\\\) i.e., the set of integers \\\\({0,1,...,m-1}\\\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler's phi function, which is \\\\(\\Phi(m)\\\\)\n\n***\nlet m have the following canonical factorization\n\\\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot ... \\cdot p_{n}^{e_n}\\\\]\nwhere the \\\\(p_i\\\\) are distinct prime numbers and \\\\( e_i\\\\) are positive integers, then\n\n\\\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\\\]\n***\nit is important to stress that we need to know the factoorization of m in order to calculate Euler's phi function.\n\n## Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\nthe theorem can be stated in the form also,\n\\\\[ a^{p-1} \\equiv 1 \\bmod p\\\\]\nthen the inverse of an integer is,\n\\\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\\\]\nperforming the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat's Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.\n\na generatlization of Fermat's little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler's theorem.\n***\n**Euler's Theorem**\nlet \\\\(a\\\\) and \\\\(m\\\\) be integers with \\\\(gcd(a,m) = 1\\\\), then\n\\\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\\\]\n***\nsince it works modulo m, it is applicable to integer rings \\\\(Z_{m}\\\\)\n\n## key generation\n***\n**Output**: public key: \\\\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\\\)\n1. choose two large primes p and q.\n2. compute \\\\(n = p\\cdot q\\\\)\n3. compute \\\\( \\Phi(n) = (p-1)(q-1)\\\\)\n4. select the public exponent \\\\( e \\in {1,2,...,\\Phi(n)-1} \\\\) such that \n\\\\[ gcd(e,\\Phi(n)) = 1\\\\]\n5. compute the private key d such that\n\\\\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\\\]\n***\nthe condition that \\\\( gcd(e,\\Phi(n)) = 1\\\\) ensures that the inverse of \\\\(e\\\\) exists modulo \\\\(\\Phi(n)\\\\), so that there is always a private key \\\\(d\\\\).\nthe computation of key keys \\\\(d\\\\) and \\\\(e\\\\) canb e doen at once using the extended Euclidean algorith. \n\n\n## Encryption and Decryption\ntodo!()\n\n## references\n- [1] [A Method for Obtaining Digital\nSignatures and Public-Key Cryptosystems](https://web.williams.edu/Mathematics/lg5/302/RSA.pdf) ","source":"_posts/cryptography/cryptography-03-rsa.md","raw":"---\ntitle: cryptography (3) RSA cryptosystem\ndate: 2023-06-17 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n## introduction\nthe security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].\n\n## Euclidean Algorithm\ntodo!()\n## Extended Euclidean Algorithm\ntodo!()\n\n## Euler's Phi Function\nwe consider the ring \\\\( Z_m\\\\) i.e., the set of integers \\\\({0,1,...,m-1}\\\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler's phi function, which is \\\\(\\Phi(m)\\\\)\n\n***\nlet m have the following canonical factorization\n\\\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot ... \\cdot p_{n}^{e_n}\\\\]\nwhere the \\\\(p_i\\\\) are distinct prime numbers and \\\\( e_i\\\\) are positive integers, then\n\n\\\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\\\]\n***\nit is important to stress that we need to know the factoorization of m in order to calculate Euler's phi function.\n\n## Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\nthe theorem can be stated in the form also,\n\\\\[ a^{p-1} \\equiv 1 \\bmod p\\\\]\nthen the inverse of an integer is,\n\\\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\\\]\nperforming the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat's Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.\n\na generatlization of Fermat's little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler's theorem.\n***\n**Euler's Theorem**\nlet \\\\(a\\\\) and \\\\(m\\\\) be integers with \\\\(gcd(a,m) = 1\\\\), then\n\\\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\\\]\n***\nsince it works modulo m, it is applicable to integer rings \\\\(Z_{m}\\\\)\n\n## key generation\n***\n**Output**: public key: \\\\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\\\)\n1. choose two large primes p and q.\n2. compute \\\\(n = p\\cdot q\\\\)\n3. compute \\\\( \\Phi(n) = (p-1)(q-1)\\\\)\n4. select the public exponent \\\\( e \\in {1,2,...,\\Phi(n)-1} \\\\) such that \n\\\\[ gcd(e,\\Phi(n)) = 1\\\\]\n5. compute the private key d such that\n\\\\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\\\]\n***\nthe condition that \\\\( gcd(e,\\Phi(n)) = 1\\\\) ensures that the inverse of \\\\(e\\\\) exists modulo \\\\(\\Phi(n)\\\\), so that there is always a private key \\\\(d\\\\).\nthe computation of key keys \\\\(d\\\\) and \\\\(e\\\\) canb e doen at once using the extended Euclidean algorith. \n\n\n## Encryption and Decryption\ntodo!()\n\n## references\n- [1] [A Method for Obtaining Digital\nSignatures and Public-Key Cryptosystems](https://web.williams.edu/Mathematics/lg5/302/RSA.pdf) ","slug":"cryptography/cryptography-03-rsa","published":1,"updated":"2023-07-13T16:54:10.773Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpw001j6hsj9vpw0y0r","content":"\n\n\n\n

introduction

the security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].

\n

Euclidean Algorithm

todo!()

\n

Extended Euclidean Algorithm

todo!()

\n

Euler’s Phi Function

we consider the ring \\( Z_m\\) i.e., the set of integers \\({0,1,…,m-1}\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler’s phi function, which is \\(\\Phi(m)\\)

\n
\n

let m have the following canonical factorization
\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot … \\cdot p_{n}^{e_n}\\]
where the \\(p_i\\) are distinct prime numbers and \\( e_i\\) are positive integers, then

\n

\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\]

\n
\n

it is important to stress that we need to know the factoorization of m in order to calculate Euler’s phi function.

\n

Fermat’s little theorem

Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
\\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
\\[ a^{p} \\equiv a \\bmod p\\]
the theorem can be stated in the form also,
\\[ a^{p-1} \\equiv 1 \\bmod p\\]
then the inverse of an integer is,
\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\]
performing the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat’s Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.

\n

a generatlization of Fermat’s little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler’s theorem.

\n
\n

Euler’s Theorem
let \\(a\\) and \\(m\\) be integers with \\(gcd(a,m) = 1\\), then
\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\]

\n
\n

since it works modulo m, it is applicable to integer rings \\(Z_{m}\\)

\n

key generation

\n

Output: public key: \\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\)

\n
    \n
  1. choose two large primes p and q.
    2. \n
  2. compute \\(n = p\\cdot q\\)
    3. \n
  3. compute \\( \\Phi(n) = (p-1)(q-1)\\)
    4. \n
  4. select the public exponent \\( e \\in {1,2,…,\\Phi(n)-1} \\) such that
    \\[ gcd(e,\\Phi(n)) = 1\\]
    5. \n
  5. compute the private key d such that
    \\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\]
    6. \n
\n
\n

the condition that \\( gcd(e,\\Phi(n)) = 1\\) ensures that the inverse of \\(e\\) exists modulo \\(\\Phi(n)\\), so that there is always a private key \\(d\\).
the computation of key keys \\(d\\) and \\(e\\) canb e doen at once using the extended Euclidean algorith.

\n

Encryption and Decryption

todo!()

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

introduction

the security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].

\n

Euclidean Algorithm

todo!()

\n

Extended Euclidean Algorithm

todo!()

\n

Euler’s Phi Function

we consider the ring \\( Z_m\\) i.e., the set of integers \\({0,1,…,m-1}\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler’s phi function, which is \\(\\Phi(m)\\)

\n
\n

let m have the following canonical factorization
\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot … \\cdot p_{n}^{e_n}\\]
where the \\(p_i\\) are distinct prime numbers and \\( e_i\\) are positive integers, then

\n

\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\]

\n
\n

it is important to stress that we need to know the factoorization of m in order to calculate Euler’s phi function.

\n

Fermat’s little theorem

Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
\\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
\\[ a^{p} \\equiv a \\bmod p\\]
the theorem can be stated in the form also,
\\[ a^{p-1} \\equiv 1 \\bmod p\\]
then the inverse of an integer is,
\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\]
performing the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat’s Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.

\n

a generatlization of Fermat’s little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler’s theorem.

\n
\n

Euler’s Theorem
let \\(a\\) and \\(m\\) be integers with \\(gcd(a,m) = 1\\), then
\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\]

\n
\n

since it works modulo m, it is applicable to integer rings \\(Z_{m}\\)

\n

key generation

\n

Output: public key: \\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\)

\n
    \n
  1. choose two large primes p and q.
    2. \n
  2. compute \\(n = p\\cdot q\\)
    3. \n
  3. compute \\( \\Phi(n) = (p-1)(q-1)\\)
    4. \n
  4. select the public exponent \\( e \\in {1,2,…,\\Phi(n)-1} \\) such that
    \\[ gcd(e,\\Phi(n)) = 1\\]
    5. \n
  5. compute the private key d such that
    \\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\]
    6. \n
\n
\n

the condition that \\( gcd(e,\\Phi(n)) = 1\\) ensures that the inverse of \\(e\\) exists modulo \\(\\Phi(n)\\), so that there is always a private key \\(d\\).
the computation of key keys \\(d\\) and \\(e\\) canb e doen at once using the extended Euclidean algorith.

\n

Encryption and Decryption

todo!()

\n

references

\n"},{"title":"zkp a brief understanding","date":"2023-06-20T06:29:26.000Z","_content":"\n\n\n## introduction\nzk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.\n\nThe example that we will choose is a simple one: proving that you know the solution to a cubic equation: `x**3 + x + 5 == 35`\n\n## reference\n- [vitalik's blog: qap zero to hero](https://medium.com/@VitalikButerin/quadratic-arithmetic-programs-from-zero-to-hero-f6d558cea649)","source":"_posts/zkp/zkp-a-brief-understanding.md","raw":"---\ntitle: zkp a brief understanding\ndate: 2023-06-20 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n## introduction\nzk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.\n\nThe example that we will choose is a simple one: proving that you know the solution to a cubic equation: `x**3 + x + 5 == 35`\n\n## reference\n- [vitalik's blog: qap zero to hero](https://medium.com/@VitalikButerin/quadratic-arithmetic-programs-from-zero-to-hero-f6d558cea649)","slug":"zkp/zkp-a-brief-understanding","published":1,"updated":"2023-07-12T14:58:22.472Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpw001l6hsjedcxczok","content":"\n\n\n

introduction

zk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.

\n

The example that we will choose is a simple one: proving that you know the solution to a cubic equation: x**3 + x + 5 == 35

\n

reference

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

zk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.

\n

The example that we will choose is a simple one: proving that you know the solution to a cubic equation: x**3 + x + 5 == 35

\n

reference

\n"},{"title":"geth state prune","date":"2023-03-25T11:29:43.000Z","_content":"\n> **_NOTE:_** Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.\n\n## introduction\nA snap-sync'd Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.\n\n## how pruning works\nPruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn't part of the target state trie or genesis state.\n\nGeth prunes the database in three stages:\n\n1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.\n2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.\n3. Compacting database: Geth tidies up the new database to reclaim free space.\n\n## Pruning command\n```\ngeth snapshot prune-state\n```\n\n## references\n- [geth doc](https://geth.ethereum.org/docs/fundamentals/pruning)","source":"_posts/geth/tech_docs/geth.prune.md","raw":"---\ntitle: geth state prune\ndate: 2023-03-25 19:29:43\ntags: [geth]\n---\n\n> **_NOTE:_** Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.\n\n## introduction\nA snap-sync'd Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.\n\n## how pruning works\nPruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn't part of the target state trie or genesis state.\n\nGeth prunes the database in three stages:\n\n1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.\n2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.\n3. Compacting database: Geth tidies up the new database to reclaim free space.\n\n## Pruning command\n```\ngeth snapshot prune-state\n```\n\n## references\n- [geth doc](https://geth.ethereum.org/docs/fundamentals/pruning)","slug":"geth/tech_docs/geth.prune","published":1,"updated":"2023-06-21T09:51:43.628Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpx001o6hsj5m9db7qe","content":"
\n

NOTE: Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.

\n
\n

introduction

A snap-sync’d Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.

\n

how pruning works

Pruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn’t part of the target state trie or genesis state.

\n

Geth prunes the database in three stages:

\n
    \n
  1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.
    2. \n
  2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.
    3. \n
  3. Compacting database: Geth tidies up the new database to reclaim free space.
    4. \n
\n

Pruning command

1
geth snapshot prune-state
\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"
\n

NOTE: Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.

\n
\n

introduction

A snap-sync’d Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.

\n

how pruning works

Pruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn’t part of the target state trie or genesis state.

\n

Geth prunes the database in three stages:

\n
    \n
  1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.
    2. \n
  2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.
    3. \n
  3. Compacting database: Geth tidies up the new database to reclaim free space.
    4. \n
\n

Pruning command

1
geth snapshot prune-state
\n\n

references

\n"},{"title":"geth sync","date":"2023-03-18T08:29:43.000Z","_content":"\n## state \nEthereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we'd need to hash all that data from scratch (which is not efficient).\n\nInstead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).\n\n## state storage\nMPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there's an extra multiplier from there. The net result is that a single state access is expected to amplify into **25-50** random disk accesses. \n\nOf course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.\n\n## Not all accesses are created equal\nThe Merkle Patricia tree is essential for writes (matain the capability to verify data), but it's an overhead for reads. \nAn Ethereum node accesses state in a few different places:\n- When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes. \n- When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).\n- When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.\n\nif we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster. \n\n## snapshot\nGeth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.\nsnapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n). \n\n## devil's in the details\nto maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there's a mini reorg however (even a single block), we're in trouble, because there's no undo. \nTo overcome this limitation, Geth's snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.\nWhenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.\nOf course, there are lots and lots of gotchas and caveats.\n- On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.\n- Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.\n- Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don't cause disk hits.\n- Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there's a chance for an item to be in the diffs, or if we can go to disk immediately.\n- The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.\n\nThe snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap [sync algorithm](https://github.com/ethereum/devp2p/pull/145).\n\n## Consensus layer syncing\nall consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. **Geth cannot sync without being connected to a consensus client.** \nThere are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:\n\n### optimistic sync\nOptimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.\n[more details](https://github.com/ethereum/consensus-specs/blob/dev/sync/optimistic.md)\n\n### checkpoint sync\nAlternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.\n\n## archive nodes\nAn archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node's own storage. \n\nIt is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with `--syncmode snap --gcmode archive`.\n\n\n## light nodes\nA light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. **Light nodes are not currently working on proof-of-stake Ethereum.**\n\n\n\n## full node\n### full\nA full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.\n\n### snap sync (default)\nSnap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state \"on-the-fly\". The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.\n![sync mode](/images/geth/sync.mode.jpg)\n\nSnap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.\n\n\nThe state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don't really understand why regenrated state could be invalidated). This means it is also necessary to have a 'healing' phase where errors in the state are fixed. Geth regularly reports `Syncing, state heal in progress` during state healing - this informs the user that state heal has not finished.\n\nThe healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.\n\nTo summarize, snap sync progresses in the following sequence:\n\n- download and verify headers\n- download block bodies and receipts. In parallel, download raw state data and build state trie\n- heal state trie to account for newly arriving data\n\nA node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.\n\n\n\n\n\n\n\n\n\n# references\n- [geth doc on sync mode](https://geth.ethereum.org/docs/fundamentals/sync-modes)\n- [eth.org blog on snapshot acceleration](https://blog.ethereum.org/2020/07/17/ask-about-geth-snapshot-acceleration)","source":"_posts/geth/tech_docs/geth.sync.mode.md","raw":"---\ntitle: geth sync\ndate: 2023-03-18 16:29:43\ntags: [geth]\n---\n\n## state \nEthereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we'd need to hash all that data from scratch (which is not efficient).\n\nInstead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).\n\n## state storage\nMPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there's an extra multiplier from there. The net result is that a single state access is expected to amplify into **25-50** random disk accesses. \n\nOf course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.\n\n## Not all accesses are created equal\nThe Merkle Patricia tree is essential for writes (matain the capability to verify data), but it's an overhead for reads. \nAn Ethereum node accesses state in a few different places:\n- When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes. \n- When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).\n- When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.\n\nif we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster. \n\n## snapshot\nGeth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.\nsnapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n). \n\n## devil's in the details\nto maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there's a mini reorg however (even a single block), we're in trouble, because there's no undo. \nTo overcome this limitation, Geth's snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.\nWhenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.\nOf course, there are lots and lots of gotchas and caveats.\n- On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.\n- Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.\n- Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don't cause disk hits.\n- Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there's a chance for an item to be in the diffs, or if we can go to disk immediately.\n- The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.\n\nThe snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap [sync algorithm](https://github.com/ethereum/devp2p/pull/145).\n\n## Consensus layer syncing\nall consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. **Geth cannot sync without being connected to a consensus client.** \nThere are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:\n\n### optimistic sync\nOptimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.\n[more details](https://github.com/ethereum/consensus-specs/blob/dev/sync/optimistic.md)\n\n### checkpoint sync\nAlternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.\n\n## archive nodes\nAn archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node's own storage. \n\nIt is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with `--syncmode snap --gcmode archive`.\n\n\n## light nodes\nA light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. **Light nodes are not currently working on proof-of-stake Ethereum.**\n\n\n\n## full node\n### full\nA full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.\n\n### snap sync (default)\nSnap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state \"on-the-fly\". The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.\n![sync mode](/images/geth/sync.mode.jpg)\n\nSnap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.\n\n\nThe state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don't really understand why regenrated state could be invalidated). This means it is also necessary to have a 'healing' phase where errors in the state are fixed. Geth regularly reports `Syncing, state heal in progress` during state healing - this informs the user that state heal has not finished.\n\nThe healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.\n\nTo summarize, snap sync progresses in the following sequence:\n\n- download and verify headers\n- download block bodies and receipts. In parallel, download raw state data and build state trie\n- heal state trie to account for newly arriving data\n\nA node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.\n\n\n\n\n\n\n\n\n\n# references\n- [geth doc on sync mode](https://geth.ethereum.org/docs/fundamentals/sync-modes)\n- [eth.org blog on snapshot acceleration](https://blog.ethereum.org/2020/07/17/ask-about-geth-snapshot-acceleration)","slug":"geth/tech_docs/geth.sync.mode","published":1,"updated":"2023-06-21T01:03:40.833Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wpy001q6hsj0ohw8qa1","content":"

state

Ethereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we’d need to hash all that data from scratch (which is not efficient).

\n

Instead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).

\n

state storage

MPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there’s an extra multiplier from there. The net result is that a single state access is expected to amplify into 25-50 random disk accesses.

\n

Of course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.

\n

Not all accesses are created equal

The Merkle Patricia tree is essential for writes (matain the capability to verify data), but it’s an overhead for reads.
An Ethereum node accesses state in a few different places:

\n
    \n
  • When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes.
    • \n
  • When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).
    • \n
  • When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.
    • \n
\n

if we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster.

\n

snapshot

Geth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.
snapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n).

\n

devil’s in the details

to maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there’s a mini reorg however (even a single block), we’re in trouble, because there’s no undo.
To overcome this limitation, Geth’s snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.
Whenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.
Of course, there are lots and lots of gotchas and caveats.

\n
    \n
  • On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.
    • \n
  • Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.
    • \n
  • Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don’t cause disk hits.
    • \n
  • Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there’s a chance for an item to be in the diffs, or if we can go to disk immediately.
    • \n
  • The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.
    • \n
\n

The snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap sync algorithm.

\n

Consensus layer syncing

all consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. Geth cannot sync without being connected to a consensus client.
There are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:

\n

optimistic sync

Optimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.
more details

\n

checkpoint sync

Alternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.

\n

archive nodes

An archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node’s own storage.

\n

It is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with --syncmode snap --gcmode archive.

\n

light nodes

A light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. Light nodes are not currently working on proof-of-stake Ethereum.

\n

full node

full

A full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.

\n

snap sync (default)

Snap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state “on-the-fly”. The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.
\"sync

\n

Snap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.

\n

The state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don’t really understand why regenrated state could be invalidated). This means it is also necessary to have a ‘healing’ phase where errors in the state are fixed. Geth regularly reports Syncing, state heal in progress during state healing - this informs the user that state heal has not finished.

\n

The healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.

\n

To summarize, snap sync progresses in the following sequence:

\n
    \n
  • download and verify headers
    • \n
  • download block bodies and receipts. In parallel, download raw state data and build state trie
    • \n
  • heal state trie to account for newly arriving data
    • \n
\n

A node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

state

Ethereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we’d need to hash all that data from scratch (which is not efficient).

\n

Instead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).

\n

state storage

MPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there’s an extra multiplier from there. The net result is that a single state access is expected to amplify into 25-50 random disk accesses.

\n

Of course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.

\n

Not all accesses are created equal

The Merkle Patricia tree is essential for writes (matain the capability to verify data), but it’s an overhead for reads.
An Ethereum node accesses state in a few different places:

\n
    \n
  • When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes.
    • \n
  • When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).
    • \n
  • When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.
    • \n
\n

if we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster.

\n

snapshot

Geth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.
snapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n).

\n

devil’s in the details

to maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there’s a mini reorg however (even a single block), we’re in trouble, because there’s no undo.
To overcome this limitation, Geth’s snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.
Whenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.
Of course, there are lots and lots of gotchas and caveats.

\n
    \n
  • On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.
    • \n
  • Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.
    • \n
  • Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don’t cause disk hits.
    • \n
  • Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there’s a chance for an item to be in the diffs, or if we can go to disk immediately.
    • \n
  • The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.
    • \n
\n

The snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap sync algorithm.

\n

Consensus layer syncing

all consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. Geth cannot sync without being connected to a consensus client.
There are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:

\n

optimistic sync

Optimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.
more details

\n

checkpoint sync

Alternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.

\n

archive nodes

An archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node’s own storage.

\n

It is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with --syncmode snap --gcmode archive.

\n

light nodes

A light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. Light nodes are not currently working on proof-of-stake Ethereum.

\n

full node

full

A full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.

\n

snap sync (default)

Snap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state “on-the-fly”. The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.
\"sync

\n

Snap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.

\n

The state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don’t really understand why regenrated state could be invalidated). This means it is also necessary to have a ‘healing’ phase where errors in the state are fixed. Geth regularly reports Syncing, state heal in progress during state healing - this informs the user that state heal has not finished.

\n

The healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.

\n

To summarize, snap sync progresses in the following sequence:

\n
    \n
  • download and verify headers
    • \n
  • download block bodies and receipts. In parallel, download raw state data and build state trie
    • \n
  • heal state trie to account for newly arriving data
    • \n
\n

A node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.

\n

references

\n"},{"title":"rust frequently used crates","date":"2022-12-13T09:15:23.000Z","_content":"\n\ntokio-trace -> tracing\n\ncontexts, multi threads\ncausality -\nstructured diagnostics ( no grep)\n\ntracing is part of tokio, tokio not requied\n\nspans: a perido of time, entered and exited\nevents: singular moment in time\nsubscriber: collect trace","source":"_posts/rust/crates/rust-frequently-used-crates.md","raw":"---\ntitle: rust frequently used crates\ndate: 2022-12-13 17:15:23\ntags: [rust]\n---\n\n\ntokio-trace -> tracing\n\ncontexts, multi threads\ncausality -\nstructured diagnostics ( no grep)\n\ntracing is part of tokio, tokio not requied\n\nspans: a perido of time, entered and exited\nevents: singular moment in time\nsubscriber: collect trace","slug":"rust/crates/rust-frequently-used-crates","published":1,"updated":"2023-05-03T09:29:19.269Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wq000266hsjd3g4hlvb","content":"

tokio-trace -> tracing

\n

contexts, multi threads
causality -
structured diagnostics ( no grep)

\n

tracing is part of tokio, tokio not requied

\n

spans: a perido of time, entered and exited
events: singular moment in time
subscriber: collect trace

\n","site":{"data":{}},"excerpt":"","more":"

tokio-trace -> tracing

\n

contexts, multi threads
causality -
structured diagnostics ( no grep)

\n

tracing is part of tokio, tokio not requied

\n

spans: a perido of time, entered and exited
events: singular moment in time
subscriber: collect trace

\n"},{"title":"two party ecdsa","date":"2023-02-07T06:29:26.000Z","_content":"\n\n\n## overview\nthis post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo's library (rust).\n\nUnlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\\\( k \\\\).\n\nIn this paper, we consider the specific case of two parties (and thus no honest majority) and con-struct a protocol that is approximately two orders of magnitude faster than the previous best.\n\n## Comparing ECDSA signing to EC-Schnorr signing\nIn both cases, the public verification key is an elliptic curve point \\\\( Q \\\\) and the private signing key is \\\\( x \\\\) such that \\\\( Q = x \\cdot G \\\\), where \\\\( G \\\\) is the generator point of an EC group of order \\\\( q \\\\).\n![schnorr ecdsa comparison](/images/two_party_ecdsa/schnorr_ecdsa_comparison.png)\n\nObserve that Schnorr signing can be easily distributed since the private key \\\\( x \\\\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\\\( s \\\\) includes \\\\( k^{-1} \\\\). Now, given shares \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 + k_2 = k \\bmod q\\\\) .It is very difficult to compute \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) such that \\\\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\\\)\n\n\ntwo-party protocols for ECDSA signing use multiplicative sharing of \\\\( x \\\\) and of \\\\( k \\\\). That is, the parties hold \\\\(x_1\\\\), \\\\(x_2\\\\) such that \\\\(x_1 \\cdot x_2 = x \\bmod q\\\\), and in each signing operation they generate \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 \\cdot k_2 = k \\bmod q\\\\). This enables them to easily compute \\\\(k^{-1}\\\\) since each party can locally compute \\\\(k_i^{\\prime} = k_i^{-1} \\bmod q\\\\), and then \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) are multiplicative shares of \\\\(k^{-1}\\\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\\\(P_1\\\\) can compute \\\\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\\\) and \\\\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\\\( P_2 \\\\) can compute \\\\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\\\), which will be an encryption of \n\n![paillier encryption](/images/two_party_ecdsa/paillier_enc.png)\n\nHowever, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\\\(k_1^{-1}\\\\) when the second party only has \\\\(R_1 = k_1 \\cdot G\\\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.\n\n## their results\n[WIP]\n\n## references\n- [original papger](https://eprint.iacr.org/2017/552.pdf)","source":"_posts/cryptography/two-party-ecdsa.md","raw":"---\ntitle: two party ecdsa\ndate: 2023-02-07 14:29:26\ntags: [cryptography,mpc,ecdsa]\n---\n\n\n\n## overview\nthis post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo's library (rust).\n\nUnlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\\\( k \\\\).\n\nIn this paper, we consider the specific case of two parties (and thus no honest majority) and con-struct a protocol that is approximately two orders of magnitude faster than the previous best.\n\n## Comparing ECDSA signing to EC-Schnorr signing\nIn both cases, the public verification key is an elliptic curve point \\\\( Q \\\\) and the private signing key is \\\\( x \\\\) such that \\\\( Q = x \\cdot G \\\\), where \\\\( G \\\\) is the generator point of an EC group of order \\\\( q \\\\).\n![schnorr ecdsa comparison](/images/two_party_ecdsa/schnorr_ecdsa_comparison.png)\n\nObserve that Schnorr signing can be easily distributed since the private key \\\\( x \\\\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\\\( s \\\\) includes \\\\( k^{-1} \\\\). Now, given shares \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 + k_2 = k \\bmod q\\\\) .It is very difficult to compute \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) such that \\\\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\\\)\n\n\ntwo-party protocols for ECDSA signing use multiplicative sharing of \\\\( x \\\\) and of \\\\( k \\\\). That is, the parties hold \\\\(x_1\\\\), \\\\(x_2\\\\) such that \\\\(x_1 \\cdot x_2 = x \\bmod q\\\\), and in each signing operation they generate \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 \\cdot k_2 = k \\bmod q\\\\). This enables them to easily compute \\\\(k^{-1}\\\\) since each party can locally compute \\\\(k_i^{\\prime} = k_i^{-1} \\bmod q\\\\), and then \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) are multiplicative shares of \\\\(k^{-1}\\\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\\\(P_1\\\\) can compute \\\\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\\\) and \\\\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\\\( P_2 \\\\) can compute \\\\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\\\), which will be an encryption of \n\n![paillier encryption](/images/two_party_ecdsa/paillier_enc.png)\n\nHowever, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\\\(k_1^{-1}\\\\) when the second party only has \\\\(R_1 = k_1 \\cdot G\\\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.\n\n## their results\n[WIP]\n\n## references\n- [original papger](https://eprint.iacr.org/2017/552.pdf)","slug":"cryptography/two-party-ecdsa","published":1,"updated":"2023-04-24T06:31:53.595Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wq000276hsjgnt73tin","content":"\n\n\n

overview

this post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo’s library (rust).

\n

Unlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\( k \\).

\n

In this paper, we consider the specific case of two parties (and thus no honest majority) and con-struct a protocol that is approximately two orders of magnitude faster than the previous best.

\n

Comparing ECDSA signing to EC-Schnorr signing

In both cases, the public verification key is an elliptic curve point \\( Q \\) and the private signing key is \\( x \\) such that \\( Q = x \\cdot G \\), where \\( G \\) is the generator point of an EC group of order \\( q \\).
\"schnorr

\n

Observe that Schnorr signing can be easily distributed since the private key \\( x \\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\( s \\) includes \\( k^{-1} \\). Now, given shares \\(k_1\\), \\(k_2\\) such that \\(k_1 + k_2 = k \\bmod q\\) .It is very difficult to compute \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) such that \\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\)

\n

two-party protocols for ECDSA signing use multiplicative sharing of \\( x \\) and of \\( k \\). That is, the parties hold \\(x_1\\), \\(x_2\\) such that \\(x_1 \\cdot x_2 = x \\bmod q\\), and in each signing operation they generate \\(k_1\\), \\(k_2\\) such that \\(k_1 \\cdot k_2 = k \\bmod q\\). This enables them to easily compute \\(k^{-1}\\) since each party can locally compute \\(k_i^{\\prime} = k_i^{-1} \\bmod q\\), and then \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) are multiplicative shares of \\(k^{-1}\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\(P_1\\) can compute \\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\) and \\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\( P_2 \\) can compute \\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\), which will be an encryption of

\n

\"paillier

\n

However, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\(k_1^{-1}\\) when the second party only has \\(R_1 = k_1 \\cdot G\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.

\n

their results

[WIP]

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

overview

this post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo’s library (rust).

\n

Unlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\( k \\).

\n

In this paper, we consider the specific case of two parties (and thus no honest majority) and con-struct a protocol that is approximately two orders of magnitude faster than the previous best.

\n

Comparing ECDSA signing to EC-Schnorr signing

In both cases, the public verification key is an elliptic curve point \\( Q \\) and the private signing key is \\( x \\) such that \\( Q = x \\cdot G \\), where \\( G \\) is the generator point of an EC group of order \\( q \\).
\"schnorr

\n

Observe that Schnorr signing can be easily distributed since the private key \\( x \\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\( s \\) includes \\( k^{-1} \\). Now, given shares \\(k_1\\), \\(k_2\\) such that \\(k_1 + k_2 = k \\bmod q\\) .It is very difficult to compute \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) such that \\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\)

\n

two-party protocols for ECDSA signing use multiplicative sharing of \\( x \\) and of \\( k \\). That is, the parties hold \\(x_1\\), \\(x_2\\) such that \\(x_1 \\cdot x_2 = x \\bmod q\\), and in each signing operation they generate \\(k_1\\), \\(k_2\\) such that \\(k_1 \\cdot k_2 = k \\bmod q\\). This enables them to easily compute \\(k^{-1}\\) since each party can locally compute \\(k_i^{\\prime} = k_i^{-1} \\bmod q\\), and then \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) are multiplicative shares of \\(k^{-1}\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\(P_1\\) can compute \\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\) and \\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\( P_2 \\) can compute \\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\), which will be an encryption of

\n

\"paillier

\n

However, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\(k_1^{-1}\\) when the second party only has \\(R_1 = k_1 \\cdot G\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.

\n

their results

[WIP]

\n

references

\n"},{"title":"rust crate serde","date":"2023-04-01T14:04:38.000Z","_content":"\n```rust\n#[serde(tag = \"filterType\")]\n#[serde(untagged)]\n#[serde(rename = \"PRICE_FILTER\")]\n#[serde(rename_all = \"camelCase\")]\n\n#[serde(with = \"string_or_float\")]\npub stop_price: f64,\n```","source":"_posts/rust/crates/rust-serde.md","raw":"---\ntitle: rust crate serde\ndate: 2023-04-01 22:04:38\ntags: [rust-crate]\n---\n\n```rust\n#[serde(tag = \"filterType\")]\n#[serde(untagged)]\n#[serde(rename = \"PRICE_FILTER\")]\n#[serde(rename_all = \"camelCase\")]\n\n#[serde(with = \"string_or_float\")]\npub stop_price: f64,\n```","slug":"rust/crates/rust-serde","published":1,"updated":"2023-06-29T15:48:46.930Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wq000296hsjhr1y1ijp","content":"
1
2
3
4
5
6
7
#[serde(tag = "filterType")]
#[serde(untagged)]
#[serde(rename = "PRICE_FILTER")]
#[serde(rename_all = "camelCase")]

#[serde(with = "string_or_float")]
pub stop_price: f64,
","site":{"data":{}},"excerpt":"","more":"
1
2
3
4
5
6
7
#[serde(tag = "filterType")]
#[serde(untagged)]
#[serde(rename = "PRICE_FILTER")]
#[serde(rename_all = "camelCase")]

#[serde(with = "string_or_float")]
pub stop_price: f64,
"},{"title":"rust std smart pointer & interior mutability","date":"2023-06-03T14:04:38.000Z","_content":"# smart pointer\n## [Rc](https://doc.rust-lang.org/std/rc/struct.Rc.html)\nA single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.\n\n\n\n# internal mutibility\n## [Cell](https://doc.rust-lang.org/stable/std/cell/struct.Cell.html)\n`Cell` enables mutation inside an immutable value. In other words, it enables `interior mutability`. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.\n### methods\n- `fn get(&self) -> T`\n- `fn set(&self, val: T)`\n- `fn swap(&self, other: &Cell)`\n- `fn replace(&self, val: T) -> T`\nReplaces the contained value with val, and returns the old contained value\n- `fn into_inner(self) -> T`\n- `const fn as_ptr(&self) -> *mut T`\n- `fn get_mut(&mut self) -> &mut T`\n- `fn from_mut(t: &mut T) -> &Cell`\n\n### traits\n```rust\nimpl !Sync for Cell // cannot be used in other threads\n```\n\n## [OnceCell](https://doc.rust-lang.org/stable/std/cell/struct.OnceCell.html)\nA cell which can be written to only once.\n### special methods\n- `fn get_or_init(&self, f: F) -> &T`\n\n## [LazyCell](https://doc.rust-lang.org/stable/std/cell/struct.LazyCell.html)\nA value which is initialized on the first access\n\n## [UnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.UnsafeCell.html#)\n`UnsafeCell` opts-out of the immutability guarantee for `&T`: a shared reference `&UnsafeCell` may point to data that is being mutated. This is called `interior mutability`.\nAll other types that allow internal mutability, such as `Cell` and `RefCell`, internally use `UnsafeCell` to wrap their data.\nNote that only the immutability guarantee for shared references is affected by `UnsafeCell`. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference). \n\n### methods\n- `pub const fn get(&self) -> *mut T`\nGets a mutable pointer to the wrapped value.\n- `pub fn get_mut(&mut self) -> &mut T`\nReturns a mutable reference to the underlying data\n- `pub const fn raw_get(this: *const UnsafeCell) -> *mut T`\nGets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:\n```rust\nuse std::cell::UnsafeCell;\nuse std::mem::MaybeUninit;\n\nlet m = MaybeUninit::>::uninit();\nunsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }\nlet uc = unsafe { m.assume_init() };\n\nassert_eq!(uc.into_inner(), 5);\n```\n- `fn into_inner(self) -> T`\nUnwraps the value, consuming the cell.\n\n## [SyncUnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.SyncUnsafeCell.html)\nThis is just an `UnsafeCell`, except it implements `Sync` if T implements Sync.\n\n## [std::cell::RefCell](https://doc.rust-lang.org/stable/std/cell/struct.RefCell.html)\nA mutable memory location with **dynamically** checked borrow rules\n- `fn borrow(&self) -> Ref<'_, T>`\n- `fn borrow_mut(&self) -> RefMut<'_, T>`\n- `fn as_ptr(&self) -> *mut T`\n\n# borrow\n## [std::borrow::Cow](https://doc.rust-lang.org/std/borrow/enum.Cow.html)\n","source":"_posts/rust/rust_std/rust-smart-pointer-and-internal-mutibility.md","raw":"---\ntitle: rust std smart pointer & interior mutability\ndate: 2023-06-03 22:04:38\ntags: [rust-std]\n---\n# smart pointer\n## [Rc](https://doc.rust-lang.org/std/rc/struct.Rc.html)\nA single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.\n\n\n\n# internal mutibility\n## [Cell](https://doc.rust-lang.org/stable/std/cell/struct.Cell.html)\n`Cell` enables mutation inside an immutable value. In other words, it enables `interior mutability`. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.\n### methods\n- `fn get(&self) -> T`\n- `fn set(&self, val: T)`\n- `fn swap(&self, other: &Cell)`\n- `fn replace(&self, val: T) -> T`\nReplaces the contained value with val, and returns the old contained value\n- `fn into_inner(self) -> T`\n- `const fn as_ptr(&self) -> *mut T`\n- `fn get_mut(&mut self) -> &mut T`\n- `fn from_mut(t: &mut T) -> &Cell`\n\n### traits\n```rust\nimpl !Sync for Cell // cannot be used in other threads\n```\n\n## [OnceCell](https://doc.rust-lang.org/stable/std/cell/struct.OnceCell.html)\nA cell which can be written to only once.\n### special methods\n- `fn get_or_init(&self, f: F) -> &T`\n\n## [LazyCell](https://doc.rust-lang.org/stable/std/cell/struct.LazyCell.html)\nA value which is initialized on the first access\n\n## [UnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.UnsafeCell.html#)\n`UnsafeCell` opts-out of the immutability guarantee for `&T`: a shared reference `&UnsafeCell` may point to data that is being mutated. This is called `interior mutability`.\nAll other types that allow internal mutability, such as `Cell` and `RefCell`, internally use `UnsafeCell` to wrap their data.\nNote that only the immutability guarantee for shared references is affected by `UnsafeCell`. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference). \n\n### methods\n- `pub const fn get(&self) -> *mut T`\nGets a mutable pointer to the wrapped value.\n- `pub fn get_mut(&mut self) -> &mut T`\nReturns a mutable reference to the underlying data\n- `pub const fn raw_get(this: *const UnsafeCell) -> *mut T`\nGets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:\n```rust\nuse std::cell::UnsafeCell;\nuse std::mem::MaybeUninit;\n\nlet m = MaybeUninit::>::uninit();\nunsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }\nlet uc = unsafe { m.assume_init() };\n\nassert_eq!(uc.into_inner(), 5);\n```\n- `fn into_inner(self) -> T`\nUnwraps the value, consuming the cell.\n\n## [SyncUnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.SyncUnsafeCell.html)\nThis is just an `UnsafeCell`, except it implements `Sync` if T implements Sync.\n\n## [std::cell::RefCell](https://doc.rust-lang.org/stable/std/cell/struct.RefCell.html)\nA mutable memory location with **dynamically** checked borrow rules\n- `fn borrow(&self) -> Ref<'_, T>`\n- `fn borrow_mut(&self) -> RefMut<'_, T>`\n- `fn as_ptr(&self) -> *mut T`\n\n# borrow\n## [std::borrow::Cow](https://doc.rust-lang.org/std/borrow/enum.Cow.html)\n","slug":"rust/rust_std/rust-smart-pointer-and-internal-mutibility","published":1,"updated":"2023-07-09T15:15:15.385Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wq1002b6hsj2xfef22t","content":"

smart pointer

Rc

A single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.

\n

internal mutibility

Cell

Cell<T> enables mutation inside an immutable value. In other words, it enables interior mutability. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.

\n

methods

    \n
  • fn get(&self) -> T
    • \n
  • fn set(&self, val: T)
    • \n
  • fn swap(&self, other: &Cell<T>)
    • \n
  • fn replace(&self, val: T) -> T
    Replaces the contained value with val, and returns the old contained value
    • \n
  • fn into_inner(self) -> T
    • \n
  • const fn as_ptr(&self) -> *mut T
    • \n
  • fn get_mut(&mut self) -> &mut T
    • \n
  • fn from_mut(t: &mut T) -> &Cell<T>
    • \n
\n

traits

1
impl<T> !Sync for Cell<T>  // cannot be used in other threads
\n\n

OnceCell

A cell which can be written to only once.

\n

special methods

    \n
  • fn get_or_init<F>(&self, f: F) -> &T
    • \n
\n

LazyCell

A value which is initialized on the first access

\n

UnsafeCell

UnsafeCell<T> opts-out of the immutability guarantee for &T: a shared reference &UnsafeCell<T> may point to data that is being mutated. This is called interior mutability.
All other types that allow internal mutability, such as Cell<T> and RefCell<T>, internally use UnsafeCell to wrap their data.
Note that only the immutability guarantee for shared references is affected by UnsafeCell. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference).

\n

methods

    \n
  • pub const fn get(&self) -> *mut T
    Gets a mutable pointer to the wrapped value.
    • \n
  • pub fn get_mut(&mut self) -> &mut T
    Returns a mutable reference to the underlying data
    • \n
  • pub const fn raw_get(this: *const UnsafeCell<T>) -> *mut T
    Gets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:
    1
    2
    3
    4
    5
    6
    7
    8
    use std::cell::UnsafeCell;
    use std::mem::MaybeUninit;

    let m = MaybeUninit::<UnsafeCell<i32>>::uninit();
    unsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }
    let uc = unsafe { m.assume_init() };

    assert_eq!(uc.into_inner(), 5);
    • \n
  • fn into_inner(self) -> T
    Unwraps the value, consuming the cell.
    • \n
\n

SyncUnsafeCell

This is just an UnsafeCell, except it implements Sync if T implements Sync.

\n

std::cell::RefCell

A mutable memory location with dynamically checked borrow rules

\n
    \n
  • fn borrow(&self) -> Ref<'_, T>
    • \n
  • fn borrow_mut(&self) -> RefMut<'_, T>
    • \n
  • fn as_ptr(&self) -> *mut T
    • \n
\n

borrow

std::borrow::Cow

","site":{"data":{}},"excerpt":"","more":"

smart pointer

Rc

A single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.

\n

internal mutibility

Cell

Cell<T> enables mutation inside an immutable value. In other words, it enables interior mutability. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.

\n

methods

    \n
  • fn get(&self) -> T
    • \n
  • fn set(&self, val: T)
    • \n
  • fn swap(&self, other: &Cell<T>)
    • \n
  • fn replace(&self, val: T) -> T
    Replaces the contained value with val, and returns the old contained value
    • \n
  • fn into_inner(self) -> T
    • \n
  • const fn as_ptr(&self) -> *mut T
    • \n
  • fn get_mut(&mut self) -> &mut T
    • \n
  • fn from_mut(t: &mut T) -> &Cell<T>
    • \n
\n

traits

1
impl<T> !Sync for Cell<T>  // cannot be used in other threads
\n\n

OnceCell

A cell which can be written to only once.

\n

special methods

    \n
  • fn get_or_init<F>(&self, f: F) -> &T
    • \n
\n

LazyCell

A value which is initialized on the first access

\n

UnsafeCell

UnsafeCell<T> opts-out of the immutability guarantee for &T: a shared reference &UnsafeCell<T> may point to data that is being mutated. This is called interior mutability.
All other types that allow internal mutability, such as Cell<T> and RefCell<T>, internally use UnsafeCell to wrap their data.
Note that only the immutability guarantee for shared references is affected by UnsafeCell. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference).

\n

methods

    \n
  • pub const fn get(&self) -> *mut T
    Gets a mutable pointer to the wrapped value.
    • \n
  • pub fn get_mut(&mut self) -> &mut T
    Returns a mutable reference to the underlying data
    • \n
  • pub const fn raw_get(this: *const UnsafeCell<T>) -> *mut T
    Gets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:
    1
    2
    3
    4
    5
    6
    7
    8
    use std::cell::UnsafeCell;
    use std::mem::MaybeUninit;

    let m = MaybeUninit::<UnsafeCell<i32>>::uninit();
    unsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }
    let uc = unsafe { m.assume_init() };

    assert_eq!(uc.into_inner(), 5);
    • \n
  • fn into_inner(self) -> T
    Unwraps the value, consuming the cell.
    • \n
\n

SyncUnsafeCell

This is just an UnsafeCell, except it implements Sync if T implements Sync.

\n

std::cell::RefCell

A mutable memory location with dynamically checked borrow rules

\n
    \n
  • fn borrow(&self) -> Ref<'_, T>
    • \n
  • fn borrow_mut(&self) -> RefMut<'_, T>
    • \n
  • fn as_ptr(&self) -> *mut T
    • \n
\n

borrow

std::borrow::Cow

"},{"title":"rust std data structure (1D)","date":"2023-05-01T14:04:38.000Z","_content":"\n## array\nA **fixed-size** array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.\n```rust\ntodo!()\n```\n\n## slice\nA **dynamically-sized view** into a contiguous sequence, [T].\n- `len()`: Returns the number of elements in the slice\n- `is_empty()`\n- `first()` Returns the first element of the slice, or `None` if it is empty.\n- `first_mut()` Returns a mutable **pointer** to the first element of the slice, or `None` if it is empty\n- `split_first()` Returns the first and all the rest of the elements of the slice, or `None` if it is empty.\n- `split_first_mut()` \n- `split_last()`\n- `split_last_mut()`\n- `last()`\n- `last_mut()`\n- `get(index: I)` Returns a reference to an element or subslice depending on the type of index.\n```rust\nlet v = [10, 40, 30];\nassert_eq!(Some(&40), v.get(1));\nassert_eq!(Some(&[10, 40][..]), v.get(0..2));\n```\n- `get_mut(index: I)`\n- `get_unchecked(index: I)` Returns a reference to an element or subslice, without doing bounds checking\n- `get_unchecked_mut(index: I)`\n- `as_ptr(&self) -> *const T` Returns a raw pointer to the slice's buffer\n```rust\nlet x = &[1, 2, 4];\nlet x_ptr = x.as_ptr();\nunsafe {\n for i in 0..x.len() {\n assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));\n }\n}\n```\n- `as_mut_ptr(&mut self) -> *mut T` \n```rust\nlet x = &mut [1, 2, 4];\nlet x_ptr = x.as_mut_ptr();\nunsafe {\n for i in 0..x.len() {\n *x_ptr.add(i) += 2;\n }\n}\nassert_eq!(x, &[3, 4, 6]);\n```\n- `as_ptr_range(&self) -> Range<*const T>` Returns the two raw pointers spanning the slice.\n```rust\npub const fn as_ptr_range(&self) -> Range<*const T> {\n let start = self.as_ptr();\n let end = unsafe { start.add(self.len()) };\n start..end\n}\n```\n- `as_mut_ptr_range(&mut self) -> Range<*mut T>`\n- `swap(&mut self, a: usize, b: usize)` Swaps two elements in the slice.\n- `reverse(&mut self)` Reverses the order of elements in the slice, in place.\n- `windows(&self, size: usize)` Returns an iterator over all contiguous windows of length `size`. The windows overlap. If the slice is shorter than `size`, the iterator returns no values.\n```rust\nlet slice = ['r', 'u', 's', 't'];\nlet mut iter = slice.windows(2);\nassert_eq!(iter.next().unwrap(), &['r', 'u']);\nassert_eq!(iter.next().unwrap(), &['u', 's']);\nassert_eq!(iter.next().unwrap(), &['s', 't']);\nassert!(iter.next().is_none());\n```\n- `chunks(&self, chunk_size: usize)` Returns an iterator over `chunk_size` elements of the slice at a time\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert_eq!(iter.next().unwrap(), &['m']);\nassert!(iter.next().is_none());\n```\n- `chunks_mut()`\n- `chunks_exact(&self, chunk_size: usize)`\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks_exact(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert!(iter.next().is_none());\nassert_eq!(iter.remainder(), &['m']);\n```\n- `as_chunks_unchecked(&self)` Splits the slice into a slice of `N`-element arrays, assuming that there's no remainder\n- `as_chunks(&self)` Splits the slice into a slice of `N`-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than `N`\n- `as_rchunks(&self)` r means reverse\n- `group_by(&self, pred: F)` Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on `slice[0]` and `slice[1]` then on `slice[1]` and `slice[2]` and so on\n```rust\n#![feature(slice_group_by)]\nlet slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];\nlet mut iter = slice.group_by(|a, b| a <= b);\nassert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3, 4][..]));\nassert_eq!(iter.next(), None);\n```\n- `split_at(&self, mid: usize)` Divides one slice into two at an index.\n- `split(&self, pred: F)` Returns an iterator over subslices separated by elements that match `pred`. The matched element is not contained in the subslices.\n- `splitn(&self, n: usize, pred: F)` \n- `contains(&self, x: &T)` Returns `true` if the slice contains an element with the given value.\n- `starts_with(&self, needle: &[T])` eturns `true` if `needle` is a prefix of the slice\n```rust\nlet v = [10, 40, 30];\nassert!(v.starts_with(&[10]));\nassert!(v.starts_with(&[10, 40]));\nassert!(!v.starts_with(&[50]));\n```\n- `ends_with(&self, needle: &[T])` \n- `strip_prefix` Returns a subslice with the prefix removed.\n```rust\nlet v = &[10, 40, 30];\nassert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));\nassert_eq!(v.strip_prefix(&[50]), None);\nlet prefix : &str = \"he\";\nassert_eq!(b\"hello\".strip_prefix(prefix.as_bytes()),\n Some(b\"llo\".as_ref()));\n```\n- `strip_suffix`\n- `binary_search(&self, x: &T)` Binary searches this slice for a given element.\n- `sort_unstable(&mut self)` Sorts the slice, but might not preserve the order of equal elements.\n- `rotate_left(&mut self, mid: usize)` Rotates the slice in-place such that the first `mid` elements of the slice move to the end while the last `self.len() - mid` elements move to the front.\n- `fill(&mut self, value: T)` Fills `self` with elements by cloning `value`.\n- `clone_from_slice(&mut self, src: &[T])` Copies the elements from `src` into `self`.\n- `copy_from_slice(&mut self, src: &[T])` \n- `is_sorted(&self)` \n- `take<'a, R: OneSidedRange>(self: &mut &'a Self, range: R)` Removes the subslice corresponding to the given range\n- `get_many_mut` Returns mutable references to many indices at once.\n```rust\n#![feature(get_many_mut)]\nlet v = &mut [1, 2, 3];\nif let Ok([a, b]) = v.get_many_mut([0, 2]) {\n *a = 413;\n *b = 612;\n}\nassert_eq!(v, &[413, 2, 612]);\n```\n\n## alloc::vec::Vec\n- `fn truncate(&mut self, len: usize)` Shortens the vector, keeping the first `len` elements and dropping the rest\n\n## std::collections::VecDeque\nA double-ended queue (deque) implemented with a growable ring buffer.\nSince VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.\n\n- `swap(&mut self, i: usize, j: usize)`\n- `reserve_exact(&mut self, additional: usize)` Reserves the minimum capacity for at least `additional` more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.\n- `reserve(&mut self, additional: usize)`\n- `shrink_to_fit(&mut self)` Shrinks the capacity of the deque as much as possible.\n- `truncate(&mut self, len: usize)` Shortens the deque, keeping the first `len` elements and dropping the rest.\n```rust\nuse std::collections::VecDeque;\nlet mut buf = VecDeque::new();\nbuf.push_back(5);\nbuf.push_back(10);\nbuf.push_back(15);\nassert_eq!(buf, [5, 10, 15]);\nbuf.truncate(1);\nassert_eq!(buf, [5]);\n```\n- `iter(&self)`\n- `as_slices(&self)`\n- `slice_ranges(&self, range: R)` Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range\n- `range(&self, range: R)` Creates an iterator that covers the specified range in the deque.\n```rust\nuse std::collections::VecDeque;\nlet deque: VecDeque<_> = [1, 2, 3].into();\nlet range = deque.range(2..).copied().collect::>();\nassert_eq!(range, [3]);\n// A full range covers all contents\nlet all = deque.range(..);\nassert_eq!(all.len(), 3);\n```\n- `drain(&mut self, range: R)` Removes the specified range from the deque in bulk, returning all removed elements as an iterator.\n- `clear(&mut self)`\n- `contains(&self, x: &T)` Returns `true` if the deque contains an element equal to the given value\n- `front(&self)` Provides a reference to the front element\n- `front_mut(&mut self)`\n- `back(&self)`\n- `back_mut(&mut self)`\n- `pop_front(&mut self)`\n- `pop_back(&mut self)`\n- `push_front(&mut self, value: T)`\n- `push_back(&mut self, value: T)`\n\n## [std::collections::LinkedList](https://doc.rust-lang.org/std/collections/struct.LinkedList.html)","source":"_posts/rust/rust_std/rust-std-data-structure-1.md","raw":"---\ntitle: rust std data structure (1D)\ndate: 2023-05-01 22:04:38\ntags: [rust-std]\n---\n\n## array\nA **fixed-size** array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.\n```rust\ntodo!()\n```\n\n## slice\nA **dynamically-sized view** into a contiguous sequence, [T].\n- `len()`: Returns the number of elements in the slice\n- `is_empty()`\n- `first()` Returns the first element of the slice, or `None` if it is empty.\n- `first_mut()` Returns a mutable **pointer** to the first element of the slice, or `None` if it is empty\n- `split_first()` Returns the first and all the rest of the elements of the slice, or `None` if it is empty.\n- `split_first_mut()` \n- `split_last()`\n- `split_last_mut()`\n- `last()`\n- `last_mut()`\n- `get(index: I)` Returns a reference to an element or subslice depending on the type of index.\n```rust\nlet v = [10, 40, 30];\nassert_eq!(Some(&40), v.get(1));\nassert_eq!(Some(&[10, 40][..]), v.get(0..2));\n```\n- `get_mut(index: I)`\n- `get_unchecked(index: I)` Returns a reference to an element or subslice, without doing bounds checking\n- `get_unchecked_mut(index: I)`\n- `as_ptr(&self) -> *const T` Returns a raw pointer to the slice's buffer\n```rust\nlet x = &[1, 2, 4];\nlet x_ptr = x.as_ptr();\nunsafe {\n for i in 0..x.len() {\n assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));\n }\n}\n```\n- `as_mut_ptr(&mut self) -> *mut T` \n```rust\nlet x = &mut [1, 2, 4];\nlet x_ptr = x.as_mut_ptr();\nunsafe {\n for i in 0..x.len() {\n *x_ptr.add(i) += 2;\n }\n}\nassert_eq!(x, &[3, 4, 6]);\n```\n- `as_ptr_range(&self) -> Range<*const T>` Returns the two raw pointers spanning the slice.\n```rust\npub const fn as_ptr_range(&self) -> Range<*const T> {\n let start = self.as_ptr();\n let end = unsafe { start.add(self.len()) };\n start..end\n}\n```\n- `as_mut_ptr_range(&mut self) -> Range<*mut T>`\n- `swap(&mut self, a: usize, b: usize)` Swaps two elements in the slice.\n- `reverse(&mut self)` Reverses the order of elements in the slice, in place.\n- `windows(&self, size: usize)` Returns an iterator over all contiguous windows of length `size`. The windows overlap. If the slice is shorter than `size`, the iterator returns no values.\n```rust\nlet slice = ['r', 'u', 's', 't'];\nlet mut iter = slice.windows(2);\nassert_eq!(iter.next().unwrap(), &['r', 'u']);\nassert_eq!(iter.next().unwrap(), &['u', 's']);\nassert_eq!(iter.next().unwrap(), &['s', 't']);\nassert!(iter.next().is_none());\n```\n- `chunks(&self, chunk_size: usize)` Returns an iterator over `chunk_size` elements of the slice at a time\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert_eq!(iter.next().unwrap(), &['m']);\nassert!(iter.next().is_none());\n```\n- `chunks_mut()`\n- `chunks_exact(&self, chunk_size: usize)`\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks_exact(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert!(iter.next().is_none());\nassert_eq!(iter.remainder(), &['m']);\n```\n- `as_chunks_unchecked(&self)` Splits the slice into a slice of `N`-element arrays, assuming that there's no remainder\n- `as_chunks(&self)` Splits the slice into a slice of `N`-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than `N`\n- `as_rchunks(&self)` r means reverse\n- `group_by(&self, pred: F)` Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on `slice[0]` and `slice[1]` then on `slice[1]` and `slice[2]` and so on\n```rust\n#![feature(slice_group_by)]\nlet slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];\nlet mut iter = slice.group_by(|a, b| a <= b);\nassert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3, 4][..]));\nassert_eq!(iter.next(), None);\n```\n- `split_at(&self, mid: usize)` Divides one slice into two at an index.\n- `split(&self, pred: F)` Returns an iterator over subslices separated by elements that match `pred`. The matched element is not contained in the subslices.\n- `splitn(&self, n: usize, pred: F)` \n- `contains(&self, x: &T)` Returns `true` if the slice contains an element with the given value.\n- `starts_with(&self, needle: &[T])` eturns `true` if `needle` is a prefix of the slice\n```rust\nlet v = [10, 40, 30];\nassert!(v.starts_with(&[10]));\nassert!(v.starts_with(&[10, 40]));\nassert!(!v.starts_with(&[50]));\n```\n- `ends_with(&self, needle: &[T])` \n- `strip_prefix` Returns a subslice with the prefix removed.\n```rust\nlet v = &[10, 40, 30];\nassert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));\nassert_eq!(v.strip_prefix(&[50]), None);\nlet prefix : &str = \"he\";\nassert_eq!(b\"hello\".strip_prefix(prefix.as_bytes()),\n Some(b\"llo\".as_ref()));\n```\n- `strip_suffix`\n- `binary_search(&self, x: &T)` Binary searches this slice for a given element.\n- `sort_unstable(&mut self)` Sorts the slice, but might not preserve the order of equal elements.\n- `rotate_left(&mut self, mid: usize)` Rotates the slice in-place such that the first `mid` elements of the slice move to the end while the last `self.len() - mid` elements move to the front.\n- `fill(&mut self, value: T)` Fills `self` with elements by cloning `value`.\n- `clone_from_slice(&mut self, src: &[T])` Copies the elements from `src` into `self`.\n- `copy_from_slice(&mut self, src: &[T])` \n- `is_sorted(&self)` \n- `take<'a, R: OneSidedRange>(self: &mut &'a Self, range: R)` Removes the subslice corresponding to the given range\n- `get_many_mut` Returns mutable references to many indices at once.\n```rust\n#![feature(get_many_mut)]\nlet v = &mut [1, 2, 3];\nif let Ok([a, b]) = v.get_many_mut([0, 2]) {\n *a = 413;\n *b = 612;\n}\nassert_eq!(v, &[413, 2, 612]);\n```\n\n## alloc::vec::Vec\n- `fn truncate(&mut self, len: usize)` Shortens the vector, keeping the first `len` elements and dropping the rest\n\n## std::collections::VecDeque\nA double-ended queue (deque) implemented with a growable ring buffer.\nSince VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.\n\n- `swap(&mut self, i: usize, j: usize)`\n- `reserve_exact(&mut self, additional: usize)` Reserves the minimum capacity for at least `additional` more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.\n- `reserve(&mut self, additional: usize)`\n- `shrink_to_fit(&mut self)` Shrinks the capacity of the deque as much as possible.\n- `truncate(&mut self, len: usize)` Shortens the deque, keeping the first `len` elements and dropping the rest.\n```rust\nuse std::collections::VecDeque;\nlet mut buf = VecDeque::new();\nbuf.push_back(5);\nbuf.push_back(10);\nbuf.push_back(15);\nassert_eq!(buf, [5, 10, 15]);\nbuf.truncate(1);\nassert_eq!(buf, [5]);\n```\n- `iter(&self)`\n- `as_slices(&self)`\n- `slice_ranges(&self, range: R)` Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range\n- `range(&self, range: R)` Creates an iterator that covers the specified range in the deque.\n```rust\nuse std::collections::VecDeque;\nlet deque: VecDeque<_> = [1, 2, 3].into();\nlet range = deque.range(2..).copied().collect::>();\nassert_eq!(range, [3]);\n// A full range covers all contents\nlet all = deque.range(..);\nassert_eq!(all.len(), 3);\n```\n- `drain(&mut self, range: R)` Removes the specified range from the deque in bulk, returning all removed elements as an iterator.\n- `clear(&mut self)`\n- `contains(&self, x: &T)` Returns `true` if the deque contains an element equal to the given value\n- `front(&self)` Provides a reference to the front element\n- `front_mut(&mut self)`\n- `back(&self)`\n- `back_mut(&mut self)`\n- `pop_front(&mut self)`\n- `pop_back(&mut self)`\n- `push_front(&mut self, value: T)`\n- `push_back(&mut self, value: T)`\n\n## [std::collections::LinkedList](https://doc.rust-lang.org/std/collections/struct.LinkedList.html)","slug":"rust/rust_std/rust-std-data-structure-1","published":1,"updated":"2023-07-11T10:18:40.048Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wq1002c6hsj7olz9y4o","content":"

array

A fixed-size array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.

\n
1
todo!()
\n\n

slice

A dynamically-sized view into a contiguous sequence, [T].

\n
    \n
  • len(): Returns the number of elements in the slice
    • \n
  • is_empty()
    • \n
  • first() Returns the first element of the slice, or None if it is empty.
    • \n
  • first_mut() Returns a mutable pointer to the first element of the slice, or None if it is empty
    • \n
  • split_first() Returns the first and all the rest of the elements of the slice, or None if it is empty.
    • \n
  • split_first_mut()
    • \n
  • split_last()
    • \n
  • split_last_mut()
    • \n
  • last()
    • \n
  • last_mut()
    • \n
  • get<I>(index: I) Returns a reference to an element or subslice depending on the type of index.
    1
    2
    3
    let v = [10, 40, 30];
    assert_eq!(Some(&40), v.get(1));
    assert_eq!(Some(&[10, 40][..]), v.get(0..2));
    • \n
  • get_mut<I>(index: I)
    • \n
  • get_unchecked<I>(index: I) Returns a reference to an element or subslice, without doing bounds checking
    • \n
  • get_unchecked_mut<I>(index: I)
    • \n
  • as_ptr(&self) -> *const T Returns a raw pointer to the slice’s buffer
    1
    2
    3
    4
    5
    6
    7
    let x = &[1, 2, 4];
    let x_ptr = x.as_ptr();
    unsafe {
        for i in 0..x.len() {
            assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));
        }
    }
    • \n
  • as_mut_ptr(&mut self) -> *mut T 
    1
    2
    3
    4
    5
    6
    7
    8
    let x = &mut [1, 2, 4];
    let x_ptr = x.as_mut_ptr();
    unsafe {
        for i in 0..x.len() {
            *x_ptr.add(i) += 2;
        }
    }
    assert_eq!(x, &[3, 4, 6]);
    • \n
  • as_ptr_range(&self) -> Range<*const T> Returns the two raw pointers spanning the slice.
    1
    2
    3
    4
    5
    pub const fn as_ptr_range(&self) -> Range<*const T> {
        let start = self.as_ptr();
        let end = unsafe { start.add(self.len()) };
        start..end
    }
    • \n
  • as_mut_ptr_range(&mut self) -> Range<*mut T>
    • \n
  • swap(&mut self, a: usize, b: usize) Swaps two elements in the slice.
    • \n
  • reverse(&mut self) Reverses the order of elements in the slice, in place.
    • \n
  • windows(&self, size: usize) Returns an iterator over all contiguous windows of length size. The windows overlap. If the slice is shorter than size, the iterator returns no values.
    1
    2
    3
    4
    5
    6
    let slice = ['r', 'u', 's', 't'];
    let mut iter = slice.windows(2);
    assert_eq!(iter.next().unwrap(), &['r', 'u']);
    assert_eq!(iter.next().unwrap(), &['u', 's']);
    assert_eq!(iter.next().unwrap(), &['s', 't']);
    assert!(iter.next().is_none());
    • \n
  • chunks(&self, chunk_size: usize) Returns an iterator over chunk_size elements of the slice at a time
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert_eq!(iter.next().unwrap(), &['m']);
    assert!(iter.next().is_none());
    • \n
  • chunks_mut()
    • \n
  • chunks_exact(&self, chunk_size: usize)
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks_exact(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert!(iter.next().is_none());
    assert_eq!(iter.remainder(), &['m']);
    • \n
  • as_chunks_unchecked<const N: usize>(&self) Splits the slice into a slice of N-element arrays, assuming that there’s no remainder
    • \n
  • as_chunks<const N: usize>(&self) Splits the slice into a slice of N-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than N
    • \n
  • as_rchunks<const N: usize>(&self) r means reverse
    • \n
  • group_by<F>(&self, pred: F) Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on slice[0] and slice[1] then on slice[1] and slice[2] and so on
    1
    2
    3
    4
    5
    6
    7
    #![feature(slice_group_by)]
    let slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];
    let mut iter = slice.group_by(|a, b| a <= b);
    assert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3, 4][..]));
    assert_eq!(iter.next(), None);
    • \n
  • split_at(&self, mid: usize) Divides one slice into two at an index.
    • \n
  • split<F>(&self, pred: F) Returns an iterator over subslices separated by elements that match pred. The matched element is not contained in the subslices.
    • \n
  • splitn<F>(&self, n: usize, pred: F)
    • \n
  • contains(&self, x: &T) Returns true if the slice contains an element with the given value.
    • \n
  • starts_with(&self, needle: &[T]) eturns true if needle is a prefix of the slice
    1
    2
    3
    4
    let v = [10, 40, 30];
    assert!(v.starts_with(&[10]));
    assert!(v.starts_with(&[10, 40]));
    assert!(!v.starts_with(&[50]));
    • \n
  • ends_with(&self, needle: &[T])
    • \n
  • strip_prefix Returns a subslice with the prefix removed.
    1
    2
    3
    4
    5
    6
    let v = &[10, 40, 30];
    assert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));
    assert_eq!(v.strip_prefix(&[50]), None);
    let prefix : &str = "he";
    assert_eq!(b"hello".strip_prefix(prefix.as_bytes()),
               Some(b"llo".as_ref()));
    • \n
  • strip_suffix
    • \n
  • binary_search(&self, x: &T) Binary searches this slice for a given element.
    • \n
  • sort_unstable(&mut self) Sorts the slice, but might not preserve the order of equal elements.
    • \n
  • rotate_left(&mut self, mid: usize) Rotates the slice in-place such that the first mid elements of the slice move to the end while the last self.len() - mid elements move to the front.
    • \n
  • fill(&mut self, value: T) Fills self with elements by cloning value.
    • \n
  • clone_from_slice(&mut self, src: &[T]) Copies the elements from src into self.
    • \n
  • copy_from_slice(&mut self, src: &[T])
    • \n
  • is_sorted(&self)
    • \n
  • take<'a, R: OneSidedRange<usize>>(self: &mut &'a Self, range: R) Removes the subslice corresponding to the given range
    • \n
  • get_many_mut<const N: usize> Returns mutable references to many indices at once.
    1
    2
    3
    4
    5
    6
    7
    #![feature(get_many_mut)]
    let v = &mut [1, 2, 3];
    if let Ok([a, b]) = v.get_many_mut([0, 2]) {
        *a = 413;
        *b = 612;
    }
    assert_eq!(v, &[413, 2, 612]);
    • \n
\n

alloc::vec::Vec

    \n
  • fn truncate(&mut self, len: usize) Shortens the vector, keeping the first len elements and dropping the rest
    • \n
\n

std::collections::VecDeque

A double-ended queue (deque) implemented with a growable ring buffer.
Since VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.

\n
    \n
  • swap(&mut self, i: usize, j: usize)
    • \n
  • reserve_exact(&mut self, additional: usize) Reserves the minimum capacity for at least additional more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.
    • \n
  • reserve(&mut self, additional: usize)
    • \n
  • shrink_to_fit(&mut self) Shrinks the capacity of the deque as much as possible.
    • \n
  • truncate(&mut self, len: usize) Shortens the deque, keeping the first len elements and dropping the rest.
    1
    2
    3
    4
    5
    6
    7
    8
    use std::collections::VecDeque;
    let mut buf = VecDeque::new();
    buf.push_back(5);
    buf.push_back(10);
    buf.push_back(15);
    assert_eq!(buf, [5, 10, 15]);
    buf.truncate(1);
    assert_eq!(buf, [5]);
    • \n
  • iter(&self)
    • \n
  • as_slices(&self)
    • \n
  • slice_ranges<R>(&self, range: R) Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range
    • \n
  • range<R>(&self, range: R) Creates an iterator that covers the specified range in the deque.
    1
    2
    3
    4
    5
    6
    7
    use std::collections::VecDeque;
    let deque: VecDeque<_> = [1, 2, 3].into();
    let range = deque.range(2..).copied().collect::<VecDeque<_>>();
    assert_eq!(range, [3]);
    // A full range covers all contents
    let all = deque.range(..);
    assert_eq!(all.len(), 3);
    • \n
  • drain<R>(&mut self, range: R) Removes the specified range from the deque in bulk, returning all removed elements as an iterator.
    • \n
  • clear(&mut self)
    • \n
  • contains(&self, x: &T) Returns true if the deque contains an element equal to the given value
    • \n
  • front(&self) Provides a reference to the front element
    • \n
  • front_mut(&mut self)
    • \n
  • back(&self)
    • \n
  • back_mut(&mut self)
    • \n
  • pop_front(&mut self)
    • \n
  • pop_back(&mut self)
    • \n
  • push_front(&mut self, value: T)
    • \n
  • push_back(&mut self, value: T)
    • \n
\n

std::collections::LinkedList

","site":{"data":{}},"excerpt":"","more":"

array

A fixed-size array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.

\n
1
todo!()
\n\n

slice

A dynamically-sized view into a contiguous sequence, [T].

\n
    \n
  • len(): Returns the number of elements in the slice
    • \n
  • is_empty()
    • \n
  • first() Returns the first element of the slice, or None if it is empty.
    • \n
  • first_mut() Returns a mutable pointer to the first element of the slice, or None if it is empty
    • \n
  • split_first() Returns the first and all the rest of the elements of the slice, or None if it is empty.
    • \n
  • split_first_mut()
    • \n
  • split_last()
    • \n
  • split_last_mut()
    • \n
  • last()
    • \n
  • last_mut()
    • \n
  • get<I>(index: I) Returns a reference to an element or subslice depending on the type of index.
    1
    2
    3
    let v = [10, 40, 30];
    assert_eq!(Some(&40), v.get(1));
    assert_eq!(Some(&[10, 40][..]), v.get(0..2));
    • \n
  • get_mut<I>(index: I)
    • \n
  • get_unchecked<I>(index: I) Returns a reference to an element or subslice, without doing bounds checking
    • \n
  • get_unchecked_mut<I>(index: I)
    • \n
  • as_ptr(&self) -> *const T Returns a raw pointer to the slice’s buffer
    1
    2
    3
    4
    5
    6
    7
    let x = &[1, 2, 4];
    let x_ptr = x.as_ptr();
    unsafe {
        for i in 0..x.len() {
            assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));
        }
    }
    • \n
  • as_mut_ptr(&mut self) -> *mut T 
    1
    2
    3
    4
    5
    6
    7
    8
    let x = &mut [1, 2, 4];
    let x_ptr = x.as_mut_ptr();
    unsafe {
        for i in 0..x.len() {
            *x_ptr.add(i) += 2;
        }
    }
    assert_eq!(x, &[3, 4, 6]);
    • \n
  • as_ptr_range(&self) -> Range<*const T> Returns the two raw pointers spanning the slice.
    1
    2
    3
    4
    5
    pub const fn as_ptr_range(&self) -> Range<*const T> {
        let start = self.as_ptr();
        let end = unsafe { start.add(self.len()) };
        start..end
    }
    • \n
  • as_mut_ptr_range(&mut self) -> Range<*mut T>
    • \n
  • swap(&mut self, a: usize, b: usize) Swaps two elements in the slice.
    • \n
  • reverse(&mut self) Reverses the order of elements in the slice, in place.
    • \n
  • windows(&self, size: usize) Returns an iterator over all contiguous windows of length size. The windows overlap. If the slice is shorter than size, the iterator returns no values.
    1
    2
    3
    4
    5
    6
    let slice = ['r', 'u', 's', 't'];
    let mut iter = slice.windows(2);
    assert_eq!(iter.next().unwrap(), &['r', 'u']);
    assert_eq!(iter.next().unwrap(), &['u', 's']);
    assert_eq!(iter.next().unwrap(), &['s', 't']);
    assert!(iter.next().is_none());
    • \n
  • chunks(&self, chunk_size: usize) Returns an iterator over chunk_size elements of the slice at a time
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert_eq!(iter.next().unwrap(), &['m']);
    assert!(iter.next().is_none());
    • \n
  • chunks_mut()
    • \n
  • chunks_exact(&self, chunk_size: usize)
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks_exact(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert!(iter.next().is_none());
    assert_eq!(iter.remainder(), &['m']);
    • \n
  • as_chunks_unchecked<const N: usize>(&self) Splits the slice into a slice of N-element arrays, assuming that there’s no remainder
    • \n
  • as_chunks<const N: usize>(&self) Splits the slice into a slice of N-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than N
    • \n
  • as_rchunks<const N: usize>(&self) r means reverse
    • \n
  • group_by<F>(&self, pred: F) Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on slice[0] and slice[1] then on slice[1] and slice[2] and so on
    1
    2
    3
    4
    5
    6
    7
    #![feature(slice_group_by)]
    let slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];
    let mut iter = slice.group_by(|a, b| a <= b);
    assert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3, 4][..]));
    assert_eq!(iter.next(), None);
    • \n
  • split_at(&self, mid: usize) Divides one slice into two at an index.
    • \n
  • split<F>(&self, pred: F) Returns an iterator over subslices separated by elements that match pred. The matched element is not contained in the subslices.
    • \n
  • splitn<F>(&self, n: usize, pred: F)
    • \n
  • contains(&self, x: &T) Returns true if the slice contains an element with the given value.
    • \n
  • starts_with(&self, needle: &[T]) eturns true if needle is a prefix of the slice
    1
    2
    3
    4
    let v = [10, 40, 30];
    assert!(v.starts_with(&[10]));
    assert!(v.starts_with(&[10, 40]));
    assert!(!v.starts_with(&[50]));
    • \n
  • ends_with(&self, needle: &[T])
    • \n
  • strip_prefix Returns a subslice with the prefix removed.
    1
    2
    3
    4
    5
    6
    let v = &[10, 40, 30];
    assert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));
    assert_eq!(v.strip_prefix(&[50]), None);
    let prefix : &str = "he";
    assert_eq!(b"hello".strip_prefix(prefix.as_bytes()),
               Some(b"llo".as_ref()));
    • \n
  • strip_suffix
    • \n
  • binary_search(&self, x: &T) Binary searches this slice for a given element.
    • \n
  • sort_unstable(&mut self) Sorts the slice, but might not preserve the order of equal elements.
    • \n
  • rotate_left(&mut self, mid: usize) Rotates the slice in-place such that the first mid elements of the slice move to the end while the last self.len() - mid elements move to the front.
    • \n
  • fill(&mut self, value: T) Fills self with elements by cloning value.
    • \n
  • clone_from_slice(&mut self, src: &[T]) Copies the elements from src into self.
    • \n
  • copy_from_slice(&mut self, src: &[T])
    • \n
  • is_sorted(&self)
    • \n
  • take<'a, R: OneSidedRange<usize>>(self: &mut &'a Self, range: R) Removes the subslice corresponding to the given range
    • \n
  • get_many_mut<const N: usize> Returns mutable references to many indices at once.
    1
    2
    3
    4
    5
    6
    7
    #![feature(get_many_mut)]
    let v = &mut [1, 2, 3];
    if let Ok([a, b]) = v.get_many_mut([0, 2]) {
        *a = 413;
        *b = 612;
    }
    assert_eq!(v, &[413, 2, 612]);
    • \n
\n

alloc::vec::Vec

    \n
  • fn truncate(&mut self, len: usize) Shortens the vector, keeping the first len elements and dropping the rest
    • \n
\n

std::collections::VecDeque

A double-ended queue (deque) implemented with a growable ring buffer.
Since VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.

\n
    \n
  • swap(&mut self, i: usize, j: usize)
    • \n
  • reserve_exact(&mut self, additional: usize) Reserves the minimum capacity for at least additional more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.
    • \n
  • reserve(&mut self, additional: usize)
    • \n
  • shrink_to_fit(&mut self) Shrinks the capacity of the deque as much as possible.
    • \n
  • truncate(&mut self, len: usize) Shortens the deque, keeping the first len elements and dropping the rest.
    1
    2
    3
    4
    5
    6
    7
    8
    use std::collections::VecDeque;
    let mut buf = VecDeque::new();
    buf.push_back(5);
    buf.push_back(10);
    buf.push_back(15);
    assert_eq!(buf, [5, 10, 15]);
    buf.truncate(1);
    assert_eq!(buf, [5]);
    • \n
  • iter(&self)
    • \n
  • as_slices(&self)
    • \n
  • slice_ranges<R>(&self, range: R) Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range
    • \n
  • range<R>(&self, range: R) Creates an iterator that covers the specified range in the deque.
    1
    2
    3
    4
    5
    6
    7
    use std::collections::VecDeque;
    let deque: VecDeque<_> = [1, 2, 3].into();
    let range = deque.range(2..).copied().collect::<VecDeque<_>>();
    assert_eq!(range, [3]);
    // A full range covers all contents
    let all = deque.range(..);
    assert_eq!(all.len(), 3);
    • \n
  • drain<R>(&mut self, range: R) Removes the specified range from the deque in bulk, returning all removed elements as an iterator.
    • \n
  • clear(&mut self)
    • \n
  • contains(&self, x: &T) Returns true if the deque contains an element equal to the given value
    • \n
  • front(&self) Provides a reference to the front element
    • \n
  • front_mut(&mut self)
    • \n
  • back(&self)
    • \n
  • back_mut(&mut self)
    • \n
  • pop_front(&mut self)
    • \n
  • pop_back(&mut self)
    • \n
  • push_front(&mut self, value: T)
    • \n
  • push_back(&mut self, value: T)
    • \n
\n

std::collections::LinkedList

"},{"title":"geth start","date":"2022-11-01T10:15:12.000Z","_content":"\n## build from source\n```\ngit clone https://github.com/ethereum/go-ethereum.git\ncd go-ethereum\nmake geth\n```\n\n## understanding geth config\ngeth config type is defined in /cmd/geth/config.go\n```go\ntype gethConfig struct {\n\tEth ethconfig.Config\n\tNode node.Config\n\tEthstats ethstatsConfig\n\tMetrics metrics.Config\n}\n```\n- **ethconfig** (eth/ethconfig/config.go)\ncontains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options\n- **nodeConfig** (node/config.go)\nrepresents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost\n- **metrics.Config** (metrics/config.go)\ncontains the configuration for the metric collection, such as InfluxDBEndpoint, etc\n- **ethstatsConfig**\nonly one URL entry\n\ngeth provides default config in the above files. user config file path is given by the below flag\n```go\nconfigFileFlag = &cli.StringFlag{\n\t\tName: \"config\",\n\t\tUsage: \"TOML configuration file\",\n\t\tCategory: flags.EthCategory,\n\t}\n```\n\nThe config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:\n```\n./geth --sepolia dumpconfig > geth-config.toml\n```\nto specify path to config file\n```\ngeth --sepolia --config geth-config.toml\n```\n\n## key configs\n- [Eth].TxLookupLimit \nNumber of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000\n- [Node].BootstrapNodes\nused to establish connectivity with the rest of the network.\ngeth provides default bootstrapNodes in file `params/bootnodes.go`\n- [Metrics_AND_STATS].ethstats\nReporting URL of a ethstats service (nodename:secret@host:port), [more detail](https://geth.ethereum.org/docs/monitoring/ethstats)\n- SyncMode\n- TrieDirtyCache\n- NoPruning\n- TrieCleanCacheJournal e.g triecache\n## how geth starts\n\n![geth starts](/images/geth_starts.drawio.png)\nthe main func is in `cmd/geth/main.go`\n```go\nfunc main() {\n\tif err := app.Run(os.Args); err != nil {\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}\n}\n```\nthe main() function is very short, and its main function is to start a tool for parsing command line commands: `gopkg.in/urfave/cli.v1`. Going deeper, we will find that `app.Action = geth` will be called when the cli app is initialized to call the geth() function\n```go\nfunc init() {\n\t// Initialize the CLI app and start Geth\n\tapp.Action = geth\n // ....\n}\n```\ngeth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.\n```go\nfunc geth(ctx *cli.Context) error {\n\tif args := ctx.Args().Slice(); len(args) > 0 {\n\t\treturn fmt.Errorf(\"invalid command: %q\", args[0])\n\t}\n\n\tprepare(ctx)\n\tstack, backend := makeFullNode(ctx)\n\tdefer stack.Close()\n\n\tstartNode(ctx, stack, backend, false)\n\tstack.Wait()\n\treturn nil\n}\n```\nIn the geth() function, there are three important function calls, namely: `prepare()`, `makeFullNode()`, and `startNode()`.\n\n### prepare\nThe implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.\n\n### makeFullNode\nThe implementation of the `makeFullNode()` function is located in the `cmd/geth/config.go` file. It will load the context of the command and apply user given configuration; and generate instances of `stack` and `backend`. Among them, `stack` is an instance of `Node` type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the `node/node.go` file), which is initialized by calling `makeConfigNode()` function through `makeFullNode()` function. inside `makeFullNode`, it calls `node.New(&cfg.Node)` to initiate a node. During instantiating of node, it invokes `rpc.NewServer()` to create a new rpc server and put in the field `inprocHandler`. it registers `rpc` api namespace by default.\n\nThe `backend` here is an interface of `ethapi.Backend` type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in `internal/ethapi/backend.go`. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. `backend` is created by calling `backend, eth := utils.RegisterEthService(stack, &cfg.Eth)`. Inside, it calls `eth.New(stack, cfg)` to create `backend` instance. During `backend` initiating, it opens database (`chainDb, err := stack.OpenDatabaseWithFreezer(\"chaindata\", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, \"eth/db/chaindata/\", false)`). Further, it creates consensus engine, `engine := ethconfig.CreateConsensusEngine(stack, ðashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb)`. goerli testnet use POA consensus (clique). \n```go\ntype Backend interface {\n\tSyncProgress() ethereum.SyncProgress\n\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)\n\tChainDb() ethdb.Database\n\tAccountManager() *accounts.Manager\n\tExtRPCEnabled() bool\n\tRPCGasCap() uint64 // global gas cap for eth_call over rpc: DoS protection\n\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection\n\tRPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs\n\tUnprotectedAllowed() bool // allows only for EIP155 transactions.\n\tSetHead(number uint64)\n\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)\n\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)\n\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)\n\tCurrentHeader() *types.Header\n\tCurrentBlock() *types.Header\n\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)\n\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)\n\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)\n\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)\n\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)\n\tPendingBlockAndReceipts() (*types.Block, types.Receipts)\n\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)\n\tGetTd(ctx context.Context, hash common.Hash) *big.Int\n\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)\n\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription\n\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription\n\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription\n\tSendTx(ctx context.Context, signedTx *types.Transaction) error\n\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)\n\tGetPoolTransactions() (types.Transactions, error)\n\tGetPoolTransaction(txHash common.Hash) *types.Transaction\n\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)\n\tStats() (pending int, queued int)\n\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)\n\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)\n\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription\n\tChainConfig() *params.ChainConfig\n\tEngine() consensus.Engine\n\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)\n\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)\n\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription\n\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tBloomStatus() (uint64, uint64)\n\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)\n}\n```\n\nIf readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend\n\n### startNode\nThe last key function, `startNode()`, is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in `Node.lifecycles` and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function\n\nAt the end of the geth() function, the function executes `stack.Wait()`, so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance\n\n## Node\nAs we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service\n```go\ntype Node struct {\n\teventmux *event.TypeMux\n\tconfig *Config\n\taccman *accounts.Manager\n\tlog log.Logger\n\tkeyDir string // key store directory\n\tkeyDirTemp bool // If true, key directory will be removed by Stop\n\tdirLock *flock.Flock // prevents concurrent use of instance directory\n\tstop chan struct{} // Channel to wait for termination notifications\n\tserver *p2p.Server // Currently running P2P networking layer\n\tstartStopLock sync.Mutex // Start/Stop are protected by an additional lock\n\tstate int // Tracks state of node lifecycle\n\n\tlock sync.Mutex\n\tlifecycles []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle\n\trpcAPIs []rpc.API // List of APIs currently provided by the node\n\thttp *httpServer //\n\tws *httpServer //\n\thttpAuth *httpServer //\n\twsAuth *httpServer //\n\tipc *ipcServer // Stores information about the ipc http server\n\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests\n\n\tdatabases map[*closeTrackingDB]struct{} // All open databases\n}\n```\n\n### close node\nAs mentioned earlier, the main thread of the entire program is blocked because of calling `stack.Wait()`. We can see that a channel called `stop` is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently\n```go\n// Wait blocks until the node is closed.\nfunc (n *Node) Wait() {\n <-n.stop\n}\n```\nWhen the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.\nIt is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.\n```go\n// doClose releases resources acquired by New(), collecting errors.\nfunc (n *Node) doClose(errs []error) error {\n // Close databases. This needs the lock because it needs to\n // synchronize with OpenDatabase*.\n n.lock.Lock()\n n.state = closedState\n errs = append(errs, n.closeDatabases()...)\n n.lock.Unlock()\n\n if err := n.accman.Close(); err != nil {\n errs = append(errs, err)\n }\n if n.keyDirTemp {\n if err := os.RemoveAll(n.keyDir); err != nil {\n errs = append(errs, err)\n }\n }\n\n // Release instance directory lock.\n n.closeDataDir()\n\n // Unblock n.Wait.\n close(n.stop)\n\n // Report any errors that might have occurred.\n switch len(errs) {\n case 0:\n return nil\n case 1:\n return errs[0]\n default:\n return fmt.Errorf(\"%v\", errs)\n }\n}\n```\n\n## Ethereum Backend\nWe can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.\n```go\ntype Ethereum struct {\n\tconfig *ethconfig.Config\n\n\t// Handlers\n\ttxPool *txpool.TxPool\n\tblockchain *core.BlockChain\n\thandler *handler\n\tethDialCandidates enode.Iterator\n\tsnapDialCandidates enode.Iterator\n\tmerger *consensus.Merger\n\n\t// DB interfaces\n\tchainDb ethdb.Database // Block chain database\n\n\teventMux *event.TypeMux\n\tengine consensus.Engine\n\taccountManager *accounts.Manager\n\n\tbloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests\n\tbloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports\n\tcloseBloomHandler chan struct{}\n\n\tAPIBackend *EthAPIBackend\n\n\tminer *miner.Miner\n\tgasPrice *big.Int\n\tetherbase common.Address\n\n\tnetworkID uint64\n\tnetRPCService *ethapi.NetAPI\n\n\tp2pServer *p2p.Server\n\n\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)\n\n\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully\n}\n```\nNodes start and stop Mining by calling `Ethereum.StartMining()` and `Ethereum.StopMining()`. Setting the profit account of Mining is achieved by calling `Ethereum.SetEtherbase()`\nHere we pay extra attention to the member variable `handler`. The definition of `handler` is in `eth/handler.go`.\nFrom a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, `downloader.Downloader` is responsible for synchronizing Block from the network, and `fetcher.TxFetcher` is responsible for synchronizing transactions from the network","source":"_posts/geth/code_analysis/geth.0.get.start.md","raw":"---\ntitle: geth start\ndate: 2022-11-01 18:15:12\ntags: [blockchain,geth]\n---\n\n## build from source\n```\ngit clone https://github.com/ethereum/go-ethereum.git\ncd go-ethereum\nmake geth\n```\n\n## understanding geth config\ngeth config type is defined in /cmd/geth/config.go\n```go\ntype gethConfig struct {\n\tEth ethconfig.Config\n\tNode node.Config\n\tEthstats ethstatsConfig\n\tMetrics metrics.Config\n}\n```\n- **ethconfig** (eth/ethconfig/config.go)\ncontains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options\n- **nodeConfig** (node/config.go)\nrepresents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost\n- **metrics.Config** (metrics/config.go)\ncontains the configuration for the metric collection, such as InfluxDBEndpoint, etc\n- **ethstatsConfig**\nonly one URL entry\n\ngeth provides default config in the above files. user config file path is given by the below flag\n```go\nconfigFileFlag = &cli.StringFlag{\n\t\tName: \"config\",\n\t\tUsage: \"TOML configuration file\",\n\t\tCategory: flags.EthCategory,\n\t}\n```\n\nThe config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:\n```\n./geth --sepolia dumpconfig > geth-config.toml\n```\nto specify path to config file\n```\ngeth --sepolia --config geth-config.toml\n```\n\n## key configs\n- [Eth].TxLookupLimit \nNumber of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000\n- [Node].BootstrapNodes\nused to establish connectivity with the rest of the network.\ngeth provides default bootstrapNodes in file `params/bootnodes.go`\n- [Metrics_AND_STATS].ethstats\nReporting URL of a ethstats service (nodename:secret@host:port), [more detail](https://geth.ethereum.org/docs/monitoring/ethstats)\n- SyncMode\n- TrieDirtyCache\n- NoPruning\n- TrieCleanCacheJournal e.g triecache\n## how geth starts\n\n![geth starts](/images/geth_starts.drawio.png)\nthe main func is in `cmd/geth/main.go`\n```go\nfunc main() {\n\tif err := app.Run(os.Args); err != nil {\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}\n}\n```\nthe main() function is very short, and its main function is to start a tool for parsing command line commands: `gopkg.in/urfave/cli.v1`. Going deeper, we will find that `app.Action = geth` will be called when the cli app is initialized to call the geth() function\n```go\nfunc init() {\n\t// Initialize the CLI app and start Geth\n\tapp.Action = geth\n // ....\n}\n```\ngeth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.\n```go\nfunc geth(ctx *cli.Context) error {\n\tif args := ctx.Args().Slice(); len(args) > 0 {\n\t\treturn fmt.Errorf(\"invalid command: %q\", args[0])\n\t}\n\n\tprepare(ctx)\n\tstack, backend := makeFullNode(ctx)\n\tdefer stack.Close()\n\n\tstartNode(ctx, stack, backend, false)\n\tstack.Wait()\n\treturn nil\n}\n```\nIn the geth() function, there are three important function calls, namely: `prepare()`, `makeFullNode()`, and `startNode()`.\n\n### prepare\nThe implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.\n\n### makeFullNode\nThe implementation of the `makeFullNode()` function is located in the `cmd/geth/config.go` file. It will load the context of the command and apply user given configuration; and generate instances of `stack` and `backend`. Among them, `stack` is an instance of `Node` type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the `node/node.go` file), which is initialized by calling `makeConfigNode()` function through `makeFullNode()` function. inside `makeFullNode`, it calls `node.New(&cfg.Node)` to initiate a node. During instantiating of node, it invokes `rpc.NewServer()` to create a new rpc server and put in the field `inprocHandler`. it registers `rpc` api namespace by default.\n\nThe `backend` here is an interface of `ethapi.Backend` type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in `internal/ethapi/backend.go`. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. `backend` is created by calling `backend, eth := utils.RegisterEthService(stack, &cfg.Eth)`. Inside, it calls `eth.New(stack, cfg)` to create `backend` instance. During `backend` initiating, it opens database (`chainDb, err := stack.OpenDatabaseWithFreezer(\"chaindata\", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, \"eth/db/chaindata/\", false)`). Further, it creates consensus engine, `engine := ethconfig.CreateConsensusEngine(stack, ðashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb)`. goerli testnet use POA consensus (clique). \n```go\ntype Backend interface {\n\tSyncProgress() ethereum.SyncProgress\n\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)\n\tChainDb() ethdb.Database\n\tAccountManager() *accounts.Manager\n\tExtRPCEnabled() bool\n\tRPCGasCap() uint64 // global gas cap for eth_call over rpc: DoS protection\n\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection\n\tRPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs\n\tUnprotectedAllowed() bool // allows only for EIP155 transactions.\n\tSetHead(number uint64)\n\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)\n\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)\n\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)\n\tCurrentHeader() *types.Header\n\tCurrentBlock() *types.Header\n\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)\n\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)\n\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)\n\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)\n\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)\n\tPendingBlockAndReceipts() (*types.Block, types.Receipts)\n\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)\n\tGetTd(ctx context.Context, hash common.Hash) *big.Int\n\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)\n\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription\n\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription\n\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription\n\tSendTx(ctx context.Context, signedTx *types.Transaction) error\n\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)\n\tGetPoolTransactions() (types.Transactions, error)\n\tGetPoolTransaction(txHash common.Hash) *types.Transaction\n\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)\n\tStats() (pending int, queued int)\n\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)\n\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)\n\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription\n\tChainConfig() *params.ChainConfig\n\tEngine() consensus.Engine\n\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)\n\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)\n\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription\n\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tBloomStatus() (uint64, uint64)\n\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)\n}\n```\n\nIf readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend\n\n### startNode\nThe last key function, `startNode()`, is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in `Node.lifecycles` and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function\n\nAt the end of the geth() function, the function executes `stack.Wait()`, so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance\n\n## Node\nAs we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service\n```go\ntype Node struct {\n\teventmux *event.TypeMux\n\tconfig *Config\n\taccman *accounts.Manager\n\tlog log.Logger\n\tkeyDir string // key store directory\n\tkeyDirTemp bool // If true, key directory will be removed by Stop\n\tdirLock *flock.Flock // prevents concurrent use of instance directory\n\tstop chan struct{} // Channel to wait for termination notifications\n\tserver *p2p.Server // Currently running P2P networking layer\n\tstartStopLock sync.Mutex // Start/Stop are protected by an additional lock\n\tstate int // Tracks state of node lifecycle\n\n\tlock sync.Mutex\n\tlifecycles []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle\n\trpcAPIs []rpc.API // List of APIs currently provided by the node\n\thttp *httpServer //\n\tws *httpServer //\n\thttpAuth *httpServer //\n\twsAuth *httpServer //\n\tipc *ipcServer // Stores information about the ipc http server\n\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests\n\n\tdatabases map[*closeTrackingDB]struct{} // All open databases\n}\n```\n\n### close node\nAs mentioned earlier, the main thread of the entire program is blocked because of calling `stack.Wait()`. We can see that a channel called `stop` is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently\n```go\n// Wait blocks until the node is closed.\nfunc (n *Node) Wait() {\n <-n.stop\n}\n```\nWhen the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.\nIt is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.\n```go\n// doClose releases resources acquired by New(), collecting errors.\nfunc (n *Node) doClose(errs []error) error {\n // Close databases. This needs the lock because it needs to\n // synchronize with OpenDatabase*.\n n.lock.Lock()\n n.state = closedState\n errs = append(errs, n.closeDatabases()...)\n n.lock.Unlock()\n\n if err := n.accman.Close(); err != nil {\n errs = append(errs, err)\n }\n if n.keyDirTemp {\n if err := os.RemoveAll(n.keyDir); err != nil {\n errs = append(errs, err)\n }\n }\n\n // Release instance directory lock.\n n.closeDataDir()\n\n // Unblock n.Wait.\n close(n.stop)\n\n // Report any errors that might have occurred.\n switch len(errs) {\n case 0:\n return nil\n case 1:\n return errs[0]\n default:\n return fmt.Errorf(\"%v\", errs)\n }\n}\n```\n\n## Ethereum Backend\nWe can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.\n```go\ntype Ethereum struct {\n\tconfig *ethconfig.Config\n\n\t// Handlers\n\ttxPool *txpool.TxPool\n\tblockchain *core.BlockChain\n\thandler *handler\n\tethDialCandidates enode.Iterator\n\tsnapDialCandidates enode.Iterator\n\tmerger *consensus.Merger\n\n\t// DB interfaces\n\tchainDb ethdb.Database // Block chain database\n\n\teventMux *event.TypeMux\n\tengine consensus.Engine\n\taccountManager *accounts.Manager\n\n\tbloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests\n\tbloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports\n\tcloseBloomHandler chan struct{}\n\n\tAPIBackend *EthAPIBackend\n\n\tminer *miner.Miner\n\tgasPrice *big.Int\n\tetherbase common.Address\n\n\tnetworkID uint64\n\tnetRPCService *ethapi.NetAPI\n\n\tp2pServer *p2p.Server\n\n\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)\n\n\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully\n}\n```\nNodes start and stop Mining by calling `Ethereum.StartMining()` and `Ethereum.StopMining()`. Setting the profit account of Mining is achieved by calling `Ethereum.SetEtherbase()`\nHere we pay extra attention to the member variable `handler`. The definition of `handler` is in `eth/handler.go`.\nFrom a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, `downloader.Downloader` is responsible for synchronizing Block from the network, and `fetcher.TxFetcher` is responsible for synchronizing transactions from the network","slug":"geth/code_analysis/geth.0.get.start","published":1,"updated":"2023-04-25T14:59:44.499Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wq1002d6hsjg0dbh2fw","content":"

build from source

1
2
3
git clone https://github.com/ethereum/go-ethereum.git
cd go-ethereum
make geth
\n\n

understanding geth config

geth config type is defined in /cmd/geth/config.go

\n
1
2
3
4
5
6
type gethConfig struct {
\tEth      ethconfig.Config
\tNode     node.Config
\tEthstats ethstatsConfig
\tMetrics  metrics.Config
}
\n
    \n
  • ethconfig (eth/ethconfig/config.go)
    contains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options
    • \n
  • nodeConfig (node/config.go)
    represents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost
    • \n
  • metrics.Config (metrics/config.go)
    contains the configuration for the metric collection, such as InfluxDBEndpoint, etc
    • \n
  • ethstatsConfig
    only one URL entry
    • \n
\n

geth provides default config in the above files. user config file path is given by the below flag

\n
1
2
3
4
5
configFileFlag = &cli.StringFlag{
\t\tName:     "config",
\t\tUsage:    "TOML configuration file",
\t\tCategory: flags.EthCategory,
\t}
\n\n

The config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:

\n
1
./geth --sepolia dumpconfig > geth-config.toml
\n

to specify path to config file

\n
1
geth --sepolia --config geth-config.toml
\n\n

key configs

    \n
  • [Eth].TxLookupLimit
    Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000
    • \n
  • [Node].BootstrapNodes
    used to establish connectivity with the rest of the network.
    geth provides default bootstrapNodes in file params/bootnodes.go
    • \n
  • [Metrics_AND_STATS].ethstats
    Reporting URL of a ethstats service (nodename:secret@host:port), more detail
    • \n
  • SyncMode
    • \n
  • TrieDirtyCache
    • \n
  • NoPruning
    • \n
  • TrieCleanCacheJournal e.g triecache
    • \n
\n

how geth starts

\"geth
the main func is in cmd/geth/main.go

\n
1
2
3
4
5
6
func main() {
\tif err := app.Run(os.Args); err != nil {
\t\tfmt.Fprintln(os.Stderr, err)
\t\tos.Exit(1)
\t}
}
\n

the main() function is very short, and its main function is to start a tool for parsing command line commands: gopkg.in/urfave/cli.v1. Going deeper, we will find that app.Action = geth will be called when the cli app is initialized to call the geth() function

\n
1
2
3
4
5
func init() {
\t// Initialize the CLI app and start Geth
\tapp.Action = geth
    // ....
}
\n

geth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
func geth(ctx *cli.Context) error {
\tif args := ctx.Args().Slice(); len(args) > 0 {
\t\treturn fmt.Errorf("invalid command: %q", args[0])
\t}

\tprepare(ctx)
\tstack, backend := makeFullNode(ctx)
\tdefer stack.Close()

\tstartNode(ctx, stack, backend, false)
\tstack.Wait()
\treturn nil
}
\n

In the geth() function, there are three important function calls, namely: prepare(), makeFullNode(), and startNode().

\n

prepare

The implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.

\n

makeFullNode

The implementation of the makeFullNode() function is located in the cmd/geth/config.go file. It will load the context of the command and apply user given configuration; and generate instances of stack and backend. Among them, stack is an instance of Node type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the node/node.go file), which is initialized by calling makeConfigNode() function through makeFullNode() function. inside makeFullNode, it calls node.New(&cfg.Node) to initiate a node. During instantiating of node, it invokes rpc.NewServer() to create a new rpc server and put in the field inprocHandler. it registers rpc api namespace by default.

\n

The backend here is an interface of ethapi.Backend type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in internal/ethapi/backend.go. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. backend is created by calling backend, eth := utils.RegisterEthService(stack, &cfg.Eth). Inside, it calls eth.New(stack, cfg) to create backend instance. During backend initiating, it opens database (chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)). Further, it creates consensus engine, engine := ethconfig.CreateConsensusEngine(stack, &ethashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb). goerli testnet use POA consensus (clique).

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
type Backend interface {
\tSyncProgress() ethereum.SyncProgress
\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)
\tChainDb() ethdb.Database
\tAccountManager() *accounts.Manager
\tExtRPCEnabled() bool
\tRPCGasCap() uint64            // global gas cap for eth_call over rpc: DoS protection
\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection
\tRPCTxFeeCap() float64         // global tx fee cap for all transaction related APIs
\tUnprotectedAllowed() bool     // allows only for EIP155 transactions.
\tSetHead(number uint64)
\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)
\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)
\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)
\tCurrentHeader() *types.Header
\tCurrentBlock() *types.Header
\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)
\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)
\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)
\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)
\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)
\tPendingBlockAndReceipts() (*types.Block, types.Receipts)
\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)
\tGetTd(ctx context.Context, hash common.Hash) *big.Int
\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)
\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription
\tSendTx(ctx context.Context, signedTx *types.Transaction) error
\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)
\tGetPoolTransactions() (types.Transactions, error)
\tGetPoolTransaction(txHash common.Hash) *types.Transaction
\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)
\tStats() (pending int, queued int)
\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)
\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)
\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
\tChainConfig() *params.ChainConfig
\tEngine() consensus.Engine
\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)
\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)
\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
\tBloomStatus() (uint64, uint64)
\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
}
\n\n

If readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend

\n

startNode

The last key function, startNode(), is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in Node.lifecycles and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function

\n

At the end of the geth() function, the function executes stack.Wait(), so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance

\n

Node

As we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
type Node struct {
\teventmux      *event.TypeMux
\tconfig        *Config
\taccman        *accounts.Manager
\tlog           log.Logger
\tkeyDir        string        // key store directory
\tkeyDirTemp    bool          // If true, key directory will be removed by Stop
\tdirLock       *flock.Flock  // prevents concurrent use of instance directory
\tstop          chan struct{} // Channel to wait for termination notifications
\tserver        *p2p.Server   // Currently running P2P networking layer
\tstartStopLock sync.Mutex    // Start/Stop are protected by an additional lock
\tstate         int           // Tracks state of node lifecycle

\tlock          sync.Mutex
\tlifecycles    []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle
\trpcAPIs       []rpc.API   // List of APIs currently provided by the node
\thttp          *httpServer //
\tws            *httpServer //
\thttpAuth      *httpServer //
\twsAuth        *httpServer //
\tipc           *ipcServer  // Stores information about the ipc http server
\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests

\tdatabases map[*closeTrackingDB]struct{} // All open databases
}
\n\n

close node

As mentioned earlier, the main thread of the entire program is blocked because of calling stack.Wait(). We can see that a channel called stop is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently

\n
1
2
3
4
// Wait blocks until the node is closed.
func (n *Node) Wait() {
 <-n.stop
}
\n

When the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.
It is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
// doClose releases resources acquired by New(), collecting errors.
func (n *Node) doClose(errs []error) error {
 // Close databases. This needs the lock because it needs to
 // synchronize with OpenDatabase*.
 n.lock.Lock()
 n.state = closedState
 errs = append(errs, n.closeDatabases()...)
 n.lock.Unlock()

 if err := n.accman.Close(); err != nil {
  errs = append(errs, err)
 }
 if n.keyDirTemp {
  if err := os.RemoveAll(n.keyDir); err != nil {
   errs = append(errs, err)
  }
 }

 // Release instance directory lock.
 n.closeDataDir()

 // Unblock n.Wait.
 close(n.stop)

 // Report any errors that might have occurred.
 switch len(errs) {
 case 0:
  return nil
 case 1:
  return errs[0]
 default:
  return fmt.Errorf("%v", errs)
 }
}
\n\n

Ethereum Backend

We can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
type Ethereum struct {
\tconfig *ethconfig.Config

\t// Handlers
\ttxPool             *txpool.TxPool
\tblockchain         *core.BlockChain
\thandler            *handler
\tethDialCandidates  enode.Iterator
\tsnapDialCandidates enode.Iterator
\tmerger             *consensus.Merger

\t// DB interfaces
\tchainDb ethdb.Database // Block chain database

\teventMux       *event.TypeMux
\tengine         consensus.Engine
\taccountManager *accounts.Manager

\tbloomRequests     chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests
\tbloomIndexer      *core.ChainIndexer             // Bloom indexer operating during block imports
\tcloseBloomHandler chan struct{}

\tAPIBackend *EthAPIBackend

\tminer     *miner.Miner
\tgasPrice  *big.Int
\tetherbase common.Address

\tnetworkID     uint64
\tnetRPCService *ethapi.NetAPI

\tp2pServer *p2p.Server

\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)

\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
}
\n

Nodes start and stop Mining by calling Ethereum.StartMining() and Ethereum.StopMining(). Setting the profit account of Mining is achieved by calling Ethereum.SetEtherbase()
Here we pay extra attention to the member variable handler. The definition of handler is in eth/handler.go.
From a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, downloader.Downloader is responsible for synchronizing Block from the network, and fetcher.TxFetcher is responsible for synchronizing transactions from the network

\n","site":{"data":{}},"excerpt":"","more":"

build from source

1
2
3
git clone https://github.com/ethereum/go-ethereum.git
cd go-ethereum
make geth
\n\n

understanding geth config

geth config type is defined in /cmd/geth/config.go

\n
1
2
3
4
5
6
type gethConfig struct {
\tEth      ethconfig.Config
\tNode     node.Config
\tEthstats ethstatsConfig
\tMetrics  metrics.Config
}
\n
    \n
  • ethconfig (eth/ethconfig/config.go)
    contains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options
    • \n
  • nodeConfig (node/config.go)
    represents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost
    • \n
  • metrics.Config (metrics/config.go)
    contains the configuration for the metric collection, such as InfluxDBEndpoint, etc
    • \n
  • ethstatsConfig
    only one URL entry
    • \n
\n

geth provides default config in the above files. user config file path is given by the below flag

\n
1
2
3
4
5
configFileFlag = &cli.StringFlag{
\t\tName:     "config",
\t\tUsage:    "TOML configuration file",
\t\tCategory: flags.EthCategory,
\t}
\n\n

The config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:

\n
1
./geth --sepolia dumpconfig > geth-config.toml
\n

to specify path to config file

\n
1
geth --sepolia --config geth-config.toml
\n\n

key configs

    \n
  • [Eth].TxLookupLimit
    Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000
    • \n
  • [Node].BootstrapNodes
    used to establish connectivity with the rest of the network.
    geth provides default bootstrapNodes in file params/bootnodes.go
    • \n
  • [Metrics_AND_STATS].ethstats
    Reporting URL of a ethstats service (nodename:secret@host:port), more detail
    • \n
  • SyncMode
    • \n
  • TrieDirtyCache
    • \n
  • NoPruning
    • \n
  • TrieCleanCacheJournal e.g triecache
    • \n
\n

how geth starts

\"geth
the main func is in cmd/geth/main.go

\n
1
2
3
4
5
6
func main() {
\tif err := app.Run(os.Args); err != nil {
\t\tfmt.Fprintln(os.Stderr, err)
\t\tos.Exit(1)
\t}
}
\n

the main() function is very short, and its main function is to start a tool for parsing command line commands: gopkg.in/urfave/cli.v1. Going deeper, we will find that app.Action = geth will be called when the cli app is initialized to call the geth() function

\n
1
2
3
4
5
func init() {
\t// Initialize the CLI app and start Geth
\tapp.Action = geth
    // ....
}
\n

geth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
func geth(ctx *cli.Context) error {
\tif args := ctx.Args().Slice(); len(args) > 0 {
\t\treturn fmt.Errorf("invalid command: %q", args[0])
\t}

\tprepare(ctx)
\tstack, backend := makeFullNode(ctx)
\tdefer stack.Close()

\tstartNode(ctx, stack, backend, false)
\tstack.Wait()
\treturn nil
}
\n

In the geth() function, there are three important function calls, namely: prepare(), makeFullNode(), and startNode().

\n

prepare

The implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.

\n

makeFullNode

The implementation of the makeFullNode() function is located in the cmd/geth/config.go file. It will load the context of the command and apply user given configuration; and generate instances of stack and backend. Among them, stack is an instance of Node type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the node/node.go file), which is initialized by calling makeConfigNode() function through makeFullNode() function. inside makeFullNode, it calls node.New(&cfg.Node) to initiate a node. During instantiating of node, it invokes rpc.NewServer() to create a new rpc server and put in the field inprocHandler. it registers rpc api namespace by default.

\n

The backend here is an interface of ethapi.Backend type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in internal/ethapi/backend.go. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. backend is created by calling backend, eth := utils.RegisterEthService(stack, &cfg.Eth). Inside, it calls eth.New(stack, cfg) to create backend instance. During backend initiating, it opens database (chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)). Further, it creates consensus engine, engine := ethconfig.CreateConsensusEngine(stack, &ethashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb). goerli testnet use POA consensus (clique).

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
type Backend interface {
\tSyncProgress() ethereum.SyncProgress
\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)
\tChainDb() ethdb.Database
\tAccountManager() *accounts.Manager
\tExtRPCEnabled() bool
\tRPCGasCap() uint64            // global gas cap for eth_call over rpc: DoS protection
\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection
\tRPCTxFeeCap() float64         // global tx fee cap for all transaction related APIs
\tUnprotectedAllowed() bool     // allows only for EIP155 transactions.
\tSetHead(number uint64)
\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)
\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)
\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)
\tCurrentHeader() *types.Header
\tCurrentBlock() *types.Header
\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)
\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)
\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)
\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)
\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)
\tPendingBlockAndReceipts() (*types.Block, types.Receipts)
\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)
\tGetTd(ctx context.Context, hash common.Hash) *big.Int
\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)
\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription
\tSendTx(ctx context.Context, signedTx *types.Transaction) error
\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)
\tGetPoolTransactions() (types.Transactions, error)
\tGetPoolTransaction(txHash common.Hash) *types.Transaction
\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)
\tStats() (pending int, queued int)
\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)
\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)
\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
\tChainConfig() *params.ChainConfig
\tEngine() consensus.Engine
\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)
\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)
\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
\tBloomStatus() (uint64, uint64)
\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
}
\n\n

If readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend

\n

startNode

The last key function, startNode(), is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in Node.lifecycles and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function

\n

At the end of the geth() function, the function executes stack.Wait(), so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance

\n

Node

As we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
type Node struct {
\teventmux      *event.TypeMux
\tconfig        *Config
\taccman        *accounts.Manager
\tlog           log.Logger
\tkeyDir        string        // key store directory
\tkeyDirTemp    bool          // If true, key directory will be removed by Stop
\tdirLock       *flock.Flock  // prevents concurrent use of instance directory
\tstop          chan struct{} // Channel to wait for termination notifications
\tserver        *p2p.Server   // Currently running P2P networking layer
\tstartStopLock sync.Mutex    // Start/Stop are protected by an additional lock
\tstate         int           // Tracks state of node lifecycle

\tlock          sync.Mutex
\tlifecycles    []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle
\trpcAPIs       []rpc.API   // List of APIs currently provided by the node
\thttp          *httpServer //
\tws            *httpServer //
\thttpAuth      *httpServer //
\twsAuth        *httpServer //
\tipc           *ipcServer  // Stores information about the ipc http server
\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests

\tdatabases map[*closeTrackingDB]struct{} // All open databases
}
\n\n

close node

As mentioned earlier, the main thread of the entire program is blocked because of calling stack.Wait(). We can see that a channel called stop is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently

\n
1
2
3
4
// Wait blocks until the node is closed.
func (n *Node) Wait() {
 <-n.stop
}
\n

When the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.
It is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
// doClose releases resources acquired by New(), collecting errors.
func (n *Node) doClose(errs []error) error {
 // Close databases. This needs the lock because it needs to
 // synchronize with OpenDatabase*.
 n.lock.Lock()
 n.state = closedState
 errs = append(errs, n.closeDatabases()...)
 n.lock.Unlock()

 if err := n.accman.Close(); err != nil {
  errs = append(errs, err)
 }
 if n.keyDirTemp {
  if err := os.RemoveAll(n.keyDir); err != nil {
   errs = append(errs, err)
  }
 }

 // Release instance directory lock.
 n.closeDataDir()

 // Unblock n.Wait.
 close(n.stop)

 // Report any errors that might have occurred.
 switch len(errs) {
 case 0:
  return nil
 case 1:
  return errs[0]
 default:
  return fmt.Errorf("%v", errs)
 }
}
\n\n

Ethereum Backend

We can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
type Ethereum struct {
\tconfig *ethconfig.Config

\t// Handlers
\ttxPool             *txpool.TxPool
\tblockchain         *core.BlockChain
\thandler            *handler
\tethDialCandidates  enode.Iterator
\tsnapDialCandidates enode.Iterator
\tmerger             *consensus.Merger

\t// DB interfaces
\tchainDb ethdb.Database // Block chain database

\teventMux       *event.TypeMux
\tengine         consensus.Engine
\taccountManager *accounts.Manager

\tbloomRequests     chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests
\tbloomIndexer      *core.ChainIndexer             // Bloom indexer operating during block imports
\tcloseBloomHandler chan struct{}

\tAPIBackend *EthAPIBackend

\tminer     *miner.Miner
\tgasPrice  *big.Int
\tetherbase common.Address

\tnetworkID     uint64
\tnetRPCService *ethapi.NetAPI

\tp2pServer *p2p.Server

\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)

\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
}
\n

Nodes start and stop Mining by calling Ethereum.StartMining() and Ethereum.StopMining(). Setting the profit account of Mining is achieved by calling Ethereum.SetEtherbase()
Here we pay extra attention to the member variable handler. The definition of handler is in eth/handler.go.
From a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, downloader.Downloader is responsible for synchronizing Block from the network, and fetcher.TxFetcher is responsible for synchronizing transactions from the network

\n"},{"title":"geth v1.10.0 summary","date":"2023-03-15T08:29:43.000Z","_content":"\n## introduction\ngeth v1.10.0 has been [released](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.0) on Mar 4 2021. this is a late summary of v1.10.0.\n\n## snapshots\nthe snapshot feature reduces the cost of accessing an account from `O(logN)` to `O(1)`. Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for `O(logN)` disk access on writes. \nProblems it solves\n- **DoS** In 2016, Ethereum sustained its worse DoS attack ever - The [Shanghai Attacks](https://2017.edcon.io/ppt/one/Martin%20Holst%20Swende_The%20%27Shanghai%20%27Attacks_EDCON.pdf) - that lasted about 2-3 months. The attack revolved around bloating Ethereum's state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.\n- **Call** Checking a smart contract's state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.\n- **Sync** There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only **96GB** of data off disk to get a new node joined into the network.\n\ndrawbacks of snapshot\n- A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie. \nuser can disable snapshot via `--snapshot=false`\n\n## snap sync\nWhen Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync). \n- **full sync** minimized trust, choosing to execute all transactions from genesis to head. \n- **fast sync** chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it's ok to download the state associated with `HEAD-64`\n\n### delays of fast sync\n- network latency (download node)\n- io latency (level db random disk access)\n- upload latency (requst with node `hash` to remote servers)\n\nThe core idea of `snap sync` is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:\n- Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.\n- Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.\n- Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO\n\n## offline pruning\nWhen processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could \"just delete\" state data that's old enough to not run the risk of a reorg. it's exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.\nIf you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run `geth snapshot prune-state`.\n\n## transaction unindexing\nNode operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It's also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.\nGeth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use `--txlookuplimit` to control the indexing block range\n\n## preimage discarding\nEthereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.\nthe preimage is the actual key related to the hash. The preimages aren't particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you'll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there's no mechanism to actively delete already stored preimages.\nIf you are using your Geth instance to debug transactions, you can retain the original behavior via `--cache.preimages`. \n\n## ETH/66 protocol\nThe eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.\n\n## chainid enforcement\nGeth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via --rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.\n\n## Database introspection\nEvery now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.\n\n## Unclean shutdown tracking\nFairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it's local chain to the point where it last saved the progress.\n\nGeth v1.10.0 will start tracking and reporting node crashes. We're hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.\n```\nWARN [03-03|06:36:38.734] Unclean shutdown detected booted=2021-02-03T06:47:28+0000 age=3w6d23h\n```\n\n## references\n- [eth foundation blog]()","source":"_posts/geth/tech_docs/geth.v1.10.0.md","raw":"---\ntitle: geth v1.10.0 summary\ndate: 2023-03-15 16:29:43\ntags: [geth]\n---\n\n## introduction\ngeth v1.10.0 has been [released](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.0) on Mar 4 2021. this is a late summary of v1.10.0.\n\n## snapshots\nthe snapshot feature reduces the cost of accessing an account from `O(logN)` to `O(1)`. Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for `O(logN)` disk access on writes. \nProblems it solves\n- **DoS** In 2016, Ethereum sustained its worse DoS attack ever - The [Shanghai Attacks](https://2017.edcon.io/ppt/one/Martin%20Holst%20Swende_The%20%27Shanghai%20%27Attacks_EDCON.pdf) - that lasted about 2-3 months. The attack revolved around bloating Ethereum's state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.\n- **Call** Checking a smart contract's state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.\n- **Sync** There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only **96GB** of data off disk to get a new node joined into the network.\n\ndrawbacks of snapshot\n- A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie. \nuser can disable snapshot via `--snapshot=false`\n\n## snap sync\nWhen Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync). \n- **full sync** minimized trust, choosing to execute all transactions from genesis to head. \n- **fast sync** chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it's ok to download the state associated with `HEAD-64`\n\n### delays of fast sync\n- network latency (download node)\n- io latency (level db random disk access)\n- upload latency (requst with node `hash` to remote servers)\n\nThe core idea of `snap sync` is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:\n- Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.\n- Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.\n- Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO\n\n## offline pruning\nWhen processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could \"just delete\" state data that's old enough to not run the risk of a reorg. it's exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.\nIf you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run `geth snapshot prune-state`.\n\n## transaction unindexing\nNode operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It's also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.\nGeth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use `--txlookuplimit` to control the indexing block range\n\n## preimage discarding\nEthereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.\nthe preimage is the actual key related to the hash. The preimages aren't particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you'll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there's no mechanism to actively delete already stored preimages.\nIf you are using your Geth instance to debug transactions, you can retain the original behavior via `--cache.preimages`. \n\n## ETH/66 protocol\nThe eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.\n\n## chainid enforcement\nGeth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via --rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.\n\n## Database introspection\nEvery now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.\n\n## Unclean shutdown tracking\nFairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it's local chain to the point where it last saved the progress.\n\nGeth v1.10.0 will start tracking and reporting node crashes. We're hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.\n```\nWARN [03-03|06:36:38.734] Unclean shutdown detected booted=2021-02-03T06:47:28+0000 age=3w6d23h\n```\n\n## references\n- [eth foundation blog]()","slug":"geth/tech_docs/geth.v1.10.0","published":1,"updated":"2023-06-18T15:06:29.245Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wq2002f6hsj4pqoevwn","content":"

introduction

geth v1.10.0 has been released on Mar 4 2021. this is a late summary of v1.10.0.

\n

snapshots

the snapshot feature reduces the cost of accessing an account from O(logN) to O(1). Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for O(logN) disk access on writes.
Problems it solves

\n
    \n
  • DoS In 2016, Ethereum sustained its worse DoS attack ever - The Shanghai Attacks - that lasted about 2-3 months. The attack revolved around bloating Ethereum’s state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.
    • \n
  • Call Checking a smart contract’s state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.
    • \n
  • Sync There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only 96GB of data off disk to get a new node joined into the network.
    • \n
\n

drawbacks of snapshot

\n
    \n
  • A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie.
    user can disable snapshot via --snapshot=false
    • \n
\n

snap sync

When Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync).

\n
    \n
  • full sync minimized trust, choosing to execute all transactions from genesis to head.
    • \n
  • fast sync chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it’s ok to download the state associated with HEAD-64
    • \n
\n

delays of fast sync

    \n
  • network latency (download node)
    • \n
  • io latency (level db random disk access)
    • \n
  • upload latency (requst with node hash to remote servers)
    • \n
\n

The core idea of snap sync is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:

\n
    \n
  • Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.
    • \n
  • Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.
    • \n
  • Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO
    • \n
\n

offline pruning

When processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could “just delete” state data that’s old enough to not run the risk of a reorg. it’s exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.
If you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run geth snapshot prune-state.

\n

transaction unindexing

Node operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It’s also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.
Geth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use --txlookuplimit to control the indexing block range

\n

preimage discarding

Ethereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.
the preimage is the actual key related to the hash. The preimages aren’t particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you’ll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there’s no mechanism to actively delete already stored preimages.
If you are using your Geth instance to debug transactions, you can retain the original behavior via --cache.preimages.

\n

ETH/66 protocol

The eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.

\n

chainid enforcement

Geth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via –rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.

\n

Database introspection

Every now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.

\n

Unclean shutdown tracking

Fairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it’s local chain to the point where it last saved the progress.

\n

Geth v1.10.0 will start tracking and reporting node crashes. We’re hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.

\n
1
WARN [03-03|06:36:38.734] Unclean shutdown detected        booted=2021-02-03T06:47:28+0000 age=3w6d23h
\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

geth v1.10.0 has been released on Mar 4 2021. this is a late summary of v1.10.0.

\n

snapshots

the snapshot feature reduces the cost of accessing an account from O(logN) to O(1). Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for O(logN) disk access on writes.
Problems it solves

\n
    \n
  • DoS In 2016, Ethereum sustained its worse DoS attack ever - The Shanghai Attacks - that lasted about 2-3 months. The attack revolved around bloating Ethereum’s state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.
    • \n
  • Call Checking a smart contract’s state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.
    • \n
  • Sync There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only 96GB of data off disk to get a new node joined into the network.
    • \n
\n

drawbacks of snapshot

\n
    \n
  • A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie.
    user can disable snapshot via --snapshot=false
    • \n
\n

snap sync

When Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync).

\n
    \n
  • full sync minimized trust, choosing to execute all transactions from genesis to head.
    • \n
  • fast sync chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it’s ok to download the state associated with HEAD-64
    • \n
\n

delays of fast sync

    \n
  • network latency (download node)
    • \n
  • io latency (level db random disk access)
    • \n
  • upload latency (requst with node hash to remote servers)
    • \n
\n

The core idea of snap sync is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:

\n
    \n
  • Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.
    • \n
  • Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.
    • \n
  • Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO
    • \n
\n

offline pruning

When processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could “just delete” state data that’s old enough to not run the risk of a reorg. it’s exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.
If you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run geth snapshot prune-state.

\n

transaction unindexing

Node operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It’s also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.
Geth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use --txlookuplimit to control the indexing block range

\n

preimage discarding

Ethereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.
the preimage is the actual key related to the hash. The preimages aren’t particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you’ll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there’s no mechanism to actively delete already stored preimages.
If you are using your Geth instance to debug transactions, you can retain the original behavior via --cache.preimages.

\n

ETH/66 protocol

The eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.

\n

chainid enforcement

Geth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via –rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.

\n

Database introspection

Every now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.

\n

Unclean shutdown tracking

Fairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it’s local chain to the point where it last saved the progress.

\n

Geth v1.10.0 will start tracking and reporting node crashes. We’re hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.

\n
1
WARN [03-03|06:36:38.734] Unclean shutdown detected        booted=2021-02-03T06:47:28+0000 age=3w6d23h
\n\n

references

\n"},{"title":"rust std sync","date":"2023-06-08T14:04:38.000Z","_content":"\n## [lock free & wait free](https://en.wikipedia.org/wiki/Non-blocking_algorithm)\n\"Lock-free\" and \"wait-free\" are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.\n- **lock-free** A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.\n- **wait-free** A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.\n\nIt's important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.\n\n## [atomic](https://doc.rust-lang.org/stable/std/sync/atomic/index.html)\nRust atomics currently follow the same rules as [C++20 atomics](https://en.cppreference.com/w/cpp/atomic), specifically `atomic_ref`. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an `atomic_ref` in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends. \nEach method takes an `Ordering` which represents the strength of the memory barrier for that operation. These orderings are the same as the [C++20 atomic orderings](https://en.cppreference.com/w/cpp/atomic/memory_order). For more information see the [nomicon](https://doc.rust-lang.org/stable/nomicon/atomics.html)\nAtomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).\n\n### Compiler Reordering\nCompilers may change the actual order of events, or make events never occur! If we write something like\n```rust\nx = 1;\ny = 3;\nx = 2;\n```\nThe compiler may conclude that it would be best if your program did:\n```rust\nx = 2;\ny = 3;\n```\nThis has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned. \n\n### Hardware Reordering\nhere is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn't actually have that memory in cache. The end result is that the hardware doesn't guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.\nFor instance, say we convince the compiler to emit this logic:\n```\ninitial state: x = 0, y = 1\n\nTHREAD 1 THREAD2\ny = 3; if x == 1 {\nx = 1; y *= 2;\n }\n```\nIdeally this program has 2 possible final states:\n- y = 3: (thread 2 did the check before thread 1 completed)\n- y = 6: (thread 2 did the check after thread 1 completed)\nHowever there's a third potential state that the hardware enables:\n- y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)\nIt's worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees. \n\n### Data Accesses\nAtomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:\n- Sequentially Consistent (SeqCst)\n- Release\n- Acquire\n- Relaxed\n\n### Sequentially Consistent\nSequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.\n\n### Acquire-Release\nAcquire and Release are largely intended to be paired. they're perfectly suited for acquiring and releasing locks. \nIntuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::thread;\n\nfn main() {\n let lock = Arc::new(AtomicBool::new(false)); // value answers \"am I locked?\"\n\n // ... distribute lock to threads somehow ...\n\n // Try to acquire the lock by setting it to true\n while lock.compare_and_swap(false, true, Ordering::Acquire) { }\n // broke out of the loop, so we successfully acquired the lock!\n\n // ... scary data accesses ...\n\n // ok we're done, release the lock\n lock.store(false, Ordering::Release);\n}\n```\n### Relaxed\nRelaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don't count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed `fetch_add` if you're not using the counter to synchronize any other accesses.\n\n## an example (spinlock)\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::{hint, thread};\n\nfn main() {\n let spinlock = Arc::new(AtomicUsize::new(1));\n\n let spinlock_clone = Arc::clone(&spinlock);\n let thread = thread::spawn(move|| {\n spinlock_clone.store(0, Ordering::SeqCst);\n });\n\n // Wait for the other thread to release the lock\n while spinlock.load(Ordering::SeqCst) != 0 {\n hint::spin_loop();\n }\n\n if let Err(panic) = thread.join() {\n println!(\"Thread had an error: {panic:?}\");\n }\n}\n```\n\n## usual structs\n1. [AtomicBool](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicBool.html)\n### methods\n- `fn get_mut(&mut self) -> &mut bool`\n- `fn into_inner(self) -> bool`\n- `fn load(&self, order: Ordering) -> bool`\n- `fn store(&self, val: bool, order: Ordering)`\n- `fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result`\nStores a value into the bool if the current value is the same as the current value.\ncompare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails. \n- `fn fetch_and(&self, val: bool, order: Ordering) -> bool`\nLogical “and” with a boolean value.\nPerforms a logical “and” operation on the current value and the argument val, and sets the new value to the result.\n- `const fn as_ptr(&self) -> *mut bool`\nReturns a mutable pointer to the underlying bool.\nDoing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.\n\n2. [AtomicUsize](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicUsize.html)\n2. [AtomicPtr](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html)\nA raw pointer type which can be safely shared between threads.\nThis type has the same in-memory representation as a *mut T\n\n\n## Higher-level synchronization objects\nMost of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.\n- **Arc**: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.\n- **Barrier**: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.\n- **Condvar**: Condition Variable, providing the ability to block a thread while waiting for an event to occur.\n- **mpsc**: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.\n- **Mutex**: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.\n- **Once**: Used for a thread-safe, one-time global initialization routine\n- **OnceLock**: Used for thread-safe, one-time initialization of a global variable.\n- **RwLock**: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.\n\n## mpsc\nThis module provides message-based communication over channels, concretely defined among three types:\n- Sender\n- SyncSender\n- Receiver","source":"_posts/rust/rust_std/rust-std-sync.md","raw":"---\ntitle: rust std sync\ndate: 2023-06-08 22:04:38\ntags: [rust-std]\n---\n\n## [lock free & wait free](https://en.wikipedia.org/wiki/Non-blocking_algorithm)\n\"Lock-free\" and \"wait-free\" are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.\n- **lock-free** A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.\n- **wait-free** A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.\n\nIt's important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.\n\n## [atomic](https://doc.rust-lang.org/stable/std/sync/atomic/index.html)\nRust atomics currently follow the same rules as [C++20 atomics](https://en.cppreference.com/w/cpp/atomic), specifically `atomic_ref`. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an `atomic_ref` in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends. \nEach method takes an `Ordering` which represents the strength of the memory barrier for that operation. These orderings are the same as the [C++20 atomic orderings](https://en.cppreference.com/w/cpp/atomic/memory_order). For more information see the [nomicon](https://doc.rust-lang.org/stable/nomicon/atomics.html)\nAtomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).\n\n### Compiler Reordering\nCompilers may change the actual order of events, or make events never occur! If we write something like\n```rust\nx = 1;\ny = 3;\nx = 2;\n```\nThe compiler may conclude that it would be best if your program did:\n```rust\nx = 2;\ny = 3;\n```\nThis has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned. \n\n### Hardware Reordering\nhere is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn't actually have that memory in cache. The end result is that the hardware doesn't guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.\nFor instance, say we convince the compiler to emit this logic:\n```\ninitial state: x = 0, y = 1\n\nTHREAD 1 THREAD2\ny = 3; if x == 1 {\nx = 1; y *= 2;\n }\n```\nIdeally this program has 2 possible final states:\n- y = 3: (thread 2 did the check before thread 1 completed)\n- y = 6: (thread 2 did the check after thread 1 completed)\nHowever there's a third potential state that the hardware enables:\n- y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)\nIt's worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees. \n\n### Data Accesses\nAtomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:\n- Sequentially Consistent (SeqCst)\n- Release\n- Acquire\n- Relaxed\n\n### Sequentially Consistent\nSequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.\n\n### Acquire-Release\nAcquire and Release are largely intended to be paired. they're perfectly suited for acquiring and releasing locks. \nIntuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::thread;\n\nfn main() {\n let lock = Arc::new(AtomicBool::new(false)); // value answers \"am I locked?\"\n\n // ... distribute lock to threads somehow ...\n\n // Try to acquire the lock by setting it to true\n while lock.compare_and_swap(false, true, Ordering::Acquire) { }\n // broke out of the loop, so we successfully acquired the lock!\n\n // ... scary data accesses ...\n\n // ok we're done, release the lock\n lock.store(false, Ordering::Release);\n}\n```\n### Relaxed\nRelaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don't count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed `fetch_add` if you're not using the counter to synchronize any other accesses.\n\n## an example (spinlock)\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::{hint, thread};\n\nfn main() {\n let spinlock = Arc::new(AtomicUsize::new(1));\n\n let spinlock_clone = Arc::clone(&spinlock);\n let thread = thread::spawn(move|| {\n spinlock_clone.store(0, Ordering::SeqCst);\n });\n\n // Wait for the other thread to release the lock\n while spinlock.load(Ordering::SeqCst) != 0 {\n hint::spin_loop();\n }\n\n if let Err(panic) = thread.join() {\n println!(\"Thread had an error: {panic:?}\");\n }\n}\n```\n\n## usual structs\n1. [AtomicBool](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicBool.html)\n### methods\n- `fn get_mut(&mut self) -> &mut bool`\n- `fn into_inner(self) -> bool`\n- `fn load(&self, order: Ordering) -> bool`\n- `fn store(&self, val: bool, order: Ordering)`\n- `fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result`\nStores a value into the bool if the current value is the same as the current value.\ncompare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails. \n- `fn fetch_and(&self, val: bool, order: Ordering) -> bool`\nLogical “and” with a boolean value.\nPerforms a logical “and” operation on the current value and the argument val, and sets the new value to the result.\n- `const fn as_ptr(&self) -> *mut bool`\nReturns a mutable pointer to the underlying bool.\nDoing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.\n\n2. [AtomicUsize](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicUsize.html)\n2. [AtomicPtr](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html)\nA raw pointer type which can be safely shared between threads.\nThis type has the same in-memory representation as a *mut T\n\n\n## Higher-level synchronization objects\nMost of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.\n- **Arc**: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.\n- **Barrier**: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.\n- **Condvar**: Condition Variable, providing the ability to block a thread while waiting for an event to occur.\n- **mpsc**: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.\n- **Mutex**: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.\n- **Once**: Used for a thread-safe, one-time global initialization routine\n- **OnceLock**: Used for thread-safe, one-time initialization of a global variable.\n- **RwLock**: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.\n\n## mpsc\nThis module provides message-based communication over channels, concretely defined among three types:\n- Sender\n- SyncSender\n- Receiver","slug":"rust/rust_std/rust-std-sync","published":1,"updated":"2023-07-05T14:21:06.619Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wq2002h6hsjd2u10w1i","content":"

lock free & wait free

“Lock-free” and “wait-free” are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.

\n
    \n
  • lock-free A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.
    • \n
  • wait-free A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.
    • \n
\n

It’s important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.

\n

atomic

Rust atomics currently follow the same rules as C++20 atomics, specifically atomic_ref. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an atomic_ref in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends.
Each method takes an Ordering which represents the strength of the memory barrier for that operation. These orderings are the same as the C++20 atomic orderings. For more information see the nomicon
Atomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).

\n

Compiler Reordering

Compilers may change the actual order of events, or make events never occur! If we write something like

\n
1
2
3
x = 1;
y = 3;
x = 2;
\n

The compiler may conclude that it would be best if your program did:

\n
1
2
x = 2;
y = 3;
\n

This has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned.

\n

Hardware Reordering

here is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn’t actually have that memory in cache. The end result is that the hardware doesn’t guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.
For instance, say we convince the compiler to emit this logic:

\n
1
2
3
4
5
6
initial state: x = 0, y = 1

THREAD 1        THREAD2
y = 3;          if x == 1 {
x = 1;              y *= 2;
                }
\n

Ideally this program has 2 possible final states:

\n
    \n
  • y = 3: (thread 2 did the check before thread 1 completed)
    • \n
  • y = 6: (thread 2 did the check after thread 1 completed)
    However there’s a third potential state that the hardware enables:
    • \n
  • y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)
    It’s worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees.
    • \n
\n

Data Accesses

Atomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:

\n
    \n
  • Sequentially Consistent (SeqCst)
    • \n
  • Release
    • \n
  • Acquire
    • \n
  • Relaxed
    • \n
\n

Sequentially Consistent

Sequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.

\n

Acquire-Release

Acquire and Release are largely intended to be paired. they’re perfectly suited for acquiring and releasing locks.
Intuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;

fn main() {
    let lock = Arc::new(AtomicBool::new(false)); // value answers "am I locked?"

    // ... distribute lock to threads somehow ...

    // Try to acquire the lock by setting it to true
    while lock.compare_and_swap(false, true, Ordering::Acquire) { }
    // broke out of the loop, so we successfully acquired the lock!

    // ... scary data accesses ...

    // ok we're done, release the lock
    lock.store(false, Ordering::Release);
}
\n

Relaxed

Relaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don’t count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed fetch_add if you’re not using the counter to synchronize any other accesses.

\n

an example (spinlock)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{hint, thread};

fn main() {
    let spinlock = Arc::new(AtomicUsize::new(1));

    let spinlock_clone = Arc::clone(&spinlock);
    let thread = thread::spawn(move|| {
        spinlock_clone.store(0, Ordering::SeqCst);
    });

    // Wait for the other thread to release the lock
    while spinlock.load(Ordering::SeqCst) != 0 {
        hint::spin_loop();
    }

    if let Err(panic) = thread.join() {
        println!("Thread had an error: {panic:?}");
    }
}
\n\n

usual structs

    \n
  1. AtomicBool
    2. \n
\n

methods

    \n
  • fn get_mut(&mut self) -> &mut bool
    • \n
  • fn into_inner(self) -> bool
    • \n
  • fn load(&self, order: Ordering) -> bool
    • \n
  • fn store(&self, val: bool, order: Ordering)
    • \n
  • fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result<bool, bool>
    Stores a value into the bool if the current value is the same as the current value.
    compare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails.
    • \n
  • fn fetch_and(&self, val: bool, order: Ordering) -> bool
    Logical “and” with a boolean value.
    Performs a logical “and” operation on the current value and the argument val, and sets the new value to the result.
    • \n
  • const fn as_ptr(&self) -> *mut bool
    Returns a mutable pointer to the underlying bool.
    Doing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.
    • \n
\n
    \n
  1. AtomicUsize
    2. \n
  2. AtomicPtr
    A raw pointer type which can be safely shared between threads.
    This type has the same in-memory representation as a *mut T
    3. \n
\n

Higher-level synchronization objects

Most of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.

\n
    \n
  • Arc: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.
    • \n
  • Barrier: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.
    • \n
  • Condvar: Condition Variable, providing the ability to block a thread while waiting for an event to occur.
    • \n
  • mpsc: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.
    • \n
  • Mutex: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.
    • \n
  • Once: Used for a thread-safe, one-time global initialization routine
    • \n
  • OnceLock: Used for thread-safe, one-time initialization of a global variable.
    • \n
  • RwLock: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.
    • \n
\n

mpsc

This module provides message-based communication over channels, concretely defined among three types:

\n
    \n
  • Sender
    • \n
  • SyncSender
    • \n
  • Receiver
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

lock free & wait free

“Lock-free” and “wait-free” are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.

\n
    \n
  • lock-free A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.
    • \n
  • wait-free A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.
    • \n
\n

It’s important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.

\n

atomic

Rust atomics currently follow the same rules as C++20 atomics, specifically atomic_ref. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an atomic_ref in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends.
Each method takes an Ordering which represents the strength of the memory barrier for that operation. These orderings are the same as the C++20 atomic orderings. For more information see the nomicon
Atomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).

\n

Compiler Reordering

Compilers may change the actual order of events, or make events never occur! If we write something like

\n
1
2
3
x = 1;
y = 3;
x = 2;
\n

The compiler may conclude that it would be best if your program did:

\n
1
2
x = 2;
y = 3;
\n

This has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned.

\n

Hardware Reordering

here is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn’t actually have that memory in cache. The end result is that the hardware doesn’t guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.
For instance, say we convince the compiler to emit this logic:

\n
1
2
3
4
5
6
initial state: x = 0, y = 1

THREAD 1        THREAD2
y = 3;          if x == 1 {
x = 1;              y *= 2;
                }
\n

Ideally this program has 2 possible final states:

\n
    \n
  • y = 3: (thread 2 did the check before thread 1 completed)
    • \n
  • y = 6: (thread 2 did the check after thread 1 completed)
    However there’s a third potential state that the hardware enables:
    • \n
  • y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)
    It’s worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees.
    • \n
\n

Data Accesses

Atomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:

\n
    \n
  • Sequentially Consistent (SeqCst)
    • \n
  • Release
    • \n
  • Acquire
    • \n
  • Relaxed
    • \n
\n

Sequentially Consistent

Sequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.

\n

Acquire-Release

Acquire and Release are largely intended to be paired. they’re perfectly suited for acquiring and releasing locks.
Intuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;

fn main() {
    let lock = Arc::new(AtomicBool::new(false)); // value answers "am I locked?"

    // ... distribute lock to threads somehow ...

    // Try to acquire the lock by setting it to true
    while lock.compare_and_swap(false, true, Ordering::Acquire) { }
    // broke out of the loop, so we successfully acquired the lock!

    // ... scary data accesses ...

    // ok we're done, release the lock
    lock.store(false, Ordering::Release);
}
\n

Relaxed

Relaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don’t count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed fetch_add if you’re not using the counter to synchronize any other accesses.

\n

an example (spinlock)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{hint, thread};

fn main() {
    let spinlock = Arc::new(AtomicUsize::new(1));

    let spinlock_clone = Arc::clone(&spinlock);
    let thread = thread::spawn(move|| {
        spinlock_clone.store(0, Ordering::SeqCst);
    });

    // Wait for the other thread to release the lock
    while spinlock.load(Ordering::SeqCst) != 0 {
        hint::spin_loop();
    }

    if let Err(panic) = thread.join() {
        println!("Thread had an error: {panic:?}");
    }
}
\n\n

usual structs

    \n
  1. AtomicBool
    2. \n
\n

methods

    \n
  • fn get_mut(&mut self) -> &mut bool
    • \n
  • fn into_inner(self) -> bool
    • \n
  • fn load(&self, order: Ordering) -> bool
    • \n
  • fn store(&self, val: bool, order: Ordering)
    • \n
  • fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result<bool, bool>
    Stores a value into the bool if the current value is the same as the current value.
    compare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails.
    • \n
  • fn fetch_and(&self, val: bool, order: Ordering) -> bool
    Logical “and” with a boolean value.
    Performs a logical “and” operation on the current value and the argument val, and sets the new value to the result.
    • \n
  • const fn as_ptr(&self) -> *mut bool
    Returns a mutable pointer to the underlying bool.
    Doing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.
    • \n
\n
    \n
  1. AtomicUsize
    2. \n
  2. AtomicPtr
    A raw pointer type which can be safely shared between threads.
    This type has the same in-memory representation as a *mut T
    3. \n
\n

Higher-level synchronization objects

Most of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.

\n
    \n
  • Arc: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.
    • \n
  • Barrier: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.
    • \n
  • Condvar: Condition Variable, providing the ability to block a thread while waiting for an event to occur.
    • \n
  • mpsc: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.
    • \n
  • Mutex: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.
    • \n
  • Once: Used for a thread-safe, one-time global initialization routine
    • \n
  • OnceLock: Used for thread-safe, one-time initialization of a global variable.
    • \n
  • RwLock: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.
    • \n
\n

mpsc

This module provides message-based communication over channels, concretely defined among three types:

\n
    \n
  • Sender
    • \n
  • SyncSender
    • \n
  • Receiver
    • \n
\n"},{"title":"rpc","date":"2022-11-08T06:23:08.000Z","_content":"\n\n## overview\npackage rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as 'services'. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.\n\n## methods\n### rpc endpoints (callback)\nMethods that satisfy the following criteria are made available for remote access:\n - method must be exported\n - method returns 0, 1 (response or error) or 2 (response and error) values\n\nThe server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.\n\nAn example server which uses the JSON codec:\n```go\ntype CalculatorService struct {}\n\nfunc (s *CalculatorService) Add(a, b int) int {\n return a + b\n}\n\nfunc (s *CalculatorService) Div(a, b int) (int, error) {\n if b == 0 {\n return 0, errors.New(\"divide by zero\")\n }\n return a/b, nil\n}\n\ncalculator := new(CalculatorService)\nserver := NewServer()\nserver.RegisterName(\"calculator\", calculator)\nl, _ := net.ListenUnix(\"unix\", &net.UnixAddr{Net: \"unix\", Name: \"/tmp/calculator.sock\"})\nserver.ServeListener(l)\n```\n\n### subscriptions\nThe package also supports the publish subscribe pattern through the use of subscriptions.\nA method that is considered eligible for notifications must satisfy the following\ncriteria:\n - method must be exported\n - first method argument type must be context.Context\n - method must have return types (rpc.Subscription, error)\n\nAn example method:\n```go\nfunc (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {\n\t\t...\n\t}\n```\n\n### Reverse Calls\nIn any method handler, an instance of rpc.Client can be accessed through the `ClientFromContext` method. Using this client instance, server-to-client method calls can be performed on the RPC connection.\n\n## server\nto start rpc service, the invoking chain is as below\n```\nnode/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]\n```\n\n### API registration\n","source":"_posts/geth/code_analysis/geth.1.rpc.md","raw":"---\ntitle: rpc\ndate: 2022-11-08 14:23:08\ntags: [blockchain, geth]\n---\n\n\n## overview\npackage rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as 'services'. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.\n\n## methods\n### rpc endpoints (callback)\nMethods that satisfy the following criteria are made available for remote access:\n - method must be exported\n - method returns 0, 1 (response or error) or 2 (response and error) values\n\nThe server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.\n\nAn example server which uses the JSON codec:\n```go\ntype CalculatorService struct {}\n\nfunc (s *CalculatorService) Add(a, b int) int {\n return a + b\n}\n\nfunc (s *CalculatorService) Div(a, b int) (int, error) {\n if b == 0 {\n return 0, errors.New(\"divide by zero\")\n }\n return a/b, nil\n}\n\ncalculator := new(CalculatorService)\nserver := NewServer()\nserver.RegisterName(\"calculator\", calculator)\nl, _ := net.ListenUnix(\"unix\", &net.UnixAddr{Net: \"unix\", Name: \"/tmp/calculator.sock\"})\nserver.ServeListener(l)\n```\n\n### subscriptions\nThe package also supports the publish subscribe pattern through the use of subscriptions.\nA method that is considered eligible for notifications must satisfy the following\ncriteria:\n - method must be exported\n - first method argument type must be context.Context\n - method must have return types (rpc.Subscription, error)\n\nAn example method:\n```go\nfunc (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {\n\t\t...\n\t}\n```\n\n### Reverse Calls\nIn any method handler, an instance of rpc.Client can be accessed through the `ClientFromContext` method. Using this client instance, server-to-client method calls can be performed on the RPC connection.\n\n## server\nto start rpc service, the invoking chain is as below\n```\nnode/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]\n```\n\n### API registration\n","slug":"geth/code_analysis/geth.1.rpc","published":1,"updated":"2023-04-27T16:54:24.069Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wq2002k6hsjamouhgu0","content":"

overview

package rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as ‘services’. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.

\n

methods

rpc endpoints (callback)

Methods that satisfy the following criteria are made available for remote access:

\n
    \n
  • method must be exported
    • \n
  • method returns 0, 1 (response or error) or 2 (response and error) values
    • \n
\n

The server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.

\n

An example server which uses the JSON codec:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
type CalculatorService struct {}

func (s *CalculatorService) Add(a, b int) int {
    return a + b
}

func (s *CalculatorService) Div(a, b int) (int, error) {
    if b == 0 {
        return 0, errors.New("divide by zero")
    }
    return a/b, nil
}

calculator := new(CalculatorService)
server := NewServer()
server.RegisterName("calculator", calculator)
l, _ := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: "/tmp/calculator.sock"})
server.ServeListener(l)
\n\n

subscriptions

The package also supports the publish subscribe pattern through the use of subscriptions.
A method that is considered eligible for notifications must satisfy the following
criteria:

\n
    \n
  • method must be exported
    • \n
  • first method argument type must be context.Context
    • \n
  • method must have return types (rpc.Subscription, error)
    • \n
\n

An example method:

\n
1
2
3
func (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {
\t\t...
\t}
\n\n

Reverse Calls

In any method handler, an instance of rpc.Client can be accessed through the ClientFromContext method. Using this client instance, server-to-client method calls can be performed on the RPC connection.

\n

server

to start rpc service, the invoking chain is as below

\n
1
node/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]
\n\n

API registration

","site":{"data":{}},"excerpt":"","more":"

overview

package rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as ‘services’. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.

\n

methods

rpc endpoints (callback)

Methods that satisfy the following criteria are made available for remote access:

\n
    \n
  • method must be exported
    • \n
  • method returns 0, 1 (response or error) or 2 (response and error) values
    • \n
\n

The server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.

\n

An example server which uses the JSON codec:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
type CalculatorService struct {}

func (s *CalculatorService) Add(a, b int) int {
    return a + b
}

func (s *CalculatorService) Div(a, b int) (int, error) {
    if b == 0 {
        return 0, errors.New("divide by zero")
    }
    return a/b, nil
}

calculator := new(CalculatorService)
server := NewServer()
server.RegisterName("calculator", calculator)
l, _ := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: "/tmp/calculator.sock"})
server.ServeListener(l)
\n\n

subscriptions

The package also supports the publish subscribe pattern through the use of subscriptions.
A method that is considered eligible for notifications must satisfy the following
criteria:

\n
    \n
  • method must be exported
    • \n
  • first method argument type must be context.Context
    • \n
  • method must have return types (rpc.Subscription, error)
    • \n
\n

An example method:

\n
1
2
3
func (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {
\t\t...
\t}
\n\n

Reverse Calls

In any method handler, an instance of rpc.Client can be accessed through the ClientFromContext method. Using this client instance, server-to-client method calls can be performed on the RPC connection.

\n

server

to start rpc service, the invoking chain is as below

\n
1
node/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]
\n\n

API registration

"},{"title":"geth evm source analysis","date":"2023-01-08T08:24:54.000Z","_content":"\n# overall\nthe code is under path `core/vm`\noverview of the whole evm module ![evm](/images/evm.drawio.google.png)\n\nthe core is `EVM` struct (in evm.go), with main function in creating or call contract. a new `EVM` object is created every time when processing a transaction. inside the EVM struct, the main items are `Interpreter`, and `StateDB` (for state persistence). `Interpreter` loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in `JumpTable` (256 sized array of `operation`)\n\ndepending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.\n\n# evm\nThe `EVM` object is the most important object exported by the evm module, which represents an Ethereum virtual machine\n\n## creating evm\nEvery time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function `ApplyTransaction` (core/state_processor.go)\n\n## creating contract\nIf the `to` of the transaction is empty, it means that this transaction is to create a contract, so call `EVM.Create` to perform related functions\n- CREATE\n```\ncontractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))\n```\n- CREATE2\n```\ncodeAndHash := &codeAndHash{code: code}\n\tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())\n```\nduring create contract, an object `Contract` is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the `jumpdests` record of the code.\n\nthen, it invokes below method to create contract\n```\nret, err := evm.interpreter.Run(contract, nil, false)\nevm.StateDB.SetCode(address, ret)\n```\nIf the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.\n\nYou may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract's \"constructor\" (that is, the contract object's constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a `ret` variable here Stored in the state database as the actual code of the contract\n\n## call contract\nThe EVM object has three methods to implement the call of the contract, they are:\n\n- EVM. Call\n- EVM. CallCode\n- EVM. DelegateCall\n- EVM.StaticCall\nThe basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods\n\n### EVM.CallCode & EVM.DelegateCall\nThe existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the \"library\" of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this \"library contract\". But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the \"library contract\", these properties must be the properties of the \"library contract\" itself, but this may not be what we want\n\nas an example\n```\nA -> contractB - delegateCall -> libC\n```\n`EVM.DelegateCall` sets the caller (msg.sender) of the \"library contract\" (libC) to A, rather than contractB; sets the address of the \"library contract\" (libC) to contractB. \n`EVM.CallCode` is similar to `EVM.DelegateCall`. the only difference is that `EVM.CallCode` only change the address of the \"library contract\" (libC) to contractB, without chanding the caller to A.\n`EVM.StaticCall` is similar to `EVM.Call`, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data\n\nduring contract call, it first check whether it is precompiled contract. some precompiled contracts are\n- common.BytesToAddress([]byte{1}): &ecrecover{},\n- common.BytesToAddress([]byte{2}): &sha256hash{},\n- common.BytesToAddress([]byte{3}): &ripemd160hash{},\n- common.BytesToAddress([]byte{4}): &dataCopy{},\n\n# EVMInterpreter\nThe interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.\n\n## execution layout\n![layout](/images/evm.layout.png)\n\n## intrinsic gas\nThe intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.\n\n## gas cost\nthe gas cost of each instruction is stored in `JumpTable.operation.dynamicGas` or `JumpTable.operation.constantGas`. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.\n\nIn fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.\n\na method `memoryGasCost`is used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.\n\n# JumpTable\njumptable is 256 sized array of `operation`\n\n## jump instruction\nAmong the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST\n```\nfunc opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {\n pos := stack.pop()\n if !contract.validJumpdest(pos) {\n nop := contract.GetOp(pos.Uint64())\n return nil, fmt.Errorf(\"invalid jump destination (%v) %v\", nop, pos)\n }\n *pc = pos.Uint64()\n\n interpreter.intPool.put(pos)\n return nil, nil\n}\n```\nA function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.\n\nTo judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.\n\nLet's introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the \"bit\" corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the \"bit\" of the offset value of the jump destination in this bit vector object is 0\n\n# references\n- [yangzhe_blog](https://yangzhe.me/2019/08/12/ethereum-evm/#%E8%A7%A3%E9%87%8A%E5%99%A8%E5%AF%B9%E8%B1%A1evminterpreter)\n- [op code manual](https://www.evm.codes/?fork=shanghai)","source":"_posts/geth/code_analysis/geth.evm.md","raw":"---\ntitle: geth evm source analysis\ndate: 2023-01-08 16:24:54\ntags: [blockchain,geth]\n---\n\n# overall\nthe code is under path `core/vm`\noverview of the whole evm module ![evm](/images/evm.drawio.google.png)\n\nthe core is `EVM` struct (in evm.go), with main function in creating or call contract. a new `EVM` object is created every time when processing a transaction. inside the EVM struct, the main items are `Interpreter`, and `StateDB` (for state persistence). `Interpreter` loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in `JumpTable` (256 sized array of `operation`)\n\ndepending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.\n\n# evm\nThe `EVM` object is the most important object exported by the evm module, which represents an Ethereum virtual machine\n\n## creating evm\nEvery time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function `ApplyTransaction` (core/state_processor.go)\n\n## creating contract\nIf the `to` of the transaction is empty, it means that this transaction is to create a contract, so call `EVM.Create` to perform related functions\n- CREATE\n```\ncontractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))\n```\n- CREATE2\n```\ncodeAndHash := &codeAndHash{code: code}\n\tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())\n```\nduring create contract, an object `Contract` is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the `jumpdests` record of the code.\n\nthen, it invokes below method to create contract\n```\nret, err := evm.interpreter.Run(contract, nil, false)\nevm.StateDB.SetCode(address, ret)\n```\nIf the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.\n\nYou may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract's \"constructor\" (that is, the contract object's constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a `ret` variable here Stored in the state database as the actual code of the contract\n\n## call contract\nThe EVM object has three methods to implement the call of the contract, they are:\n\n- EVM. Call\n- EVM. CallCode\n- EVM. DelegateCall\n- EVM.StaticCall\nThe basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods\n\n### EVM.CallCode & EVM.DelegateCall\nThe existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the \"library\" of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this \"library contract\". But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the \"library contract\", these properties must be the properties of the \"library contract\" itself, but this may not be what we want\n\nas an example\n```\nA -> contractB - delegateCall -> libC\n```\n`EVM.DelegateCall` sets the caller (msg.sender) of the \"library contract\" (libC) to A, rather than contractB; sets the address of the \"library contract\" (libC) to contractB. \n`EVM.CallCode` is similar to `EVM.DelegateCall`. the only difference is that `EVM.CallCode` only change the address of the \"library contract\" (libC) to contractB, without chanding the caller to A.\n`EVM.StaticCall` is similar to `EVM.Call`, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data\n\nduring contract call, it first check whether it is precompiled contract. some precompiled contracts are\n- common.BytesToAddress([]byte{1}): &ecrecover{},\n- common.BytesToAddress([]byte{2}): &sha256hash{},\n- common.BytesToAddress([]byte{3}): &ripemd160hash{},\n- common.BytesToAddress([]byte{4}): &dataCopy{},\n\n# EVMInterpreter\nThe interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.\n\n## execution layout\n![layout](/images/evm.layout.png)\n\n## intrinsic gas\nThe intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.\n\n## gas cost\nthe gas cost of each instruction is stored in `JumpTable.operation.dynamicGas` or `JumpTable.operation.constantGas`. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.\n\nIn fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.\n\na method `memoryGasCost`is used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.\n\n# JumpTable\njumptable is 256 sized array of `operation`\n\n## jump instruction\nAmong the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST\n```\nfunc opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {\n pos := stack.pop()\n if !contract.validJumpdest(pos) {\n nop := contract.GetOp(pos.Uint64())\n return nil, fmt.Errorf(\"invalid jump destination (%v) %v\", nop, pos)\n }\n *pc = pos.Uint64()\n\n interpreter.intPool.put(pos)\n return nil, nil\n}\n```\nA function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.\n\nTo judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.\n\nLet's introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the \"bit\" corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the \"bit\" of the offset value of the jump destination in this bit vector object is 0\n\n# references\n- [yangzhe_blog](https://yangzhe.me/2019/08/12/ethereum-evm/#%E8%A7%A3%E9%87%8A%E5%99%A8%E5%AF%B9%E8%B1%A1evminterpreter)\n- [op code manual](https://www.evm.codes/?fork=shanghai)","slug":"geth/code_analysis/geth.evm","published":1,"updated":"2023-04-14T03:02:06.144Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wq2002m6hsjg7yx2331","content":"

overall

the code is under path core/vm
overview of the whole evm module \"evm\"

\n

the core is EVM struct (in evm.go), with main function in creating or call contract. a new EVM object is created every time when processing a transaction. inside the EVM struct, the main items are Interpreter, and StateDB (for state persistence). Interpreter loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in JumpTable (256 sized array of operation)

\n

depending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.

\n

evm

The EVM object is the most important object exported by the evm module, which represents an Ethereum virtual machine

\n

creating evm

Every time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function ApplyTransaction (core/state_processor.go)

\n

creating contract

If the to of the transaction is empty, it means that this transaction is to create a contract, so call EVM.Create to perform related functions

\n
    \n
  • CREATE
    1
    contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))
    • \n
  • CREATE2
    1
    2
    codeAndHash := &codeAndHash{code: code}
    \tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())
    \nduring create contract, an object Contract is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the jumpdests record of the code.
    • \n
\n

then, it invokes below method to create contract

\n
1
2
ret, err := evm.interpreter.Run(contract, nil, false)
evm.StateDB.SetCode(address, ret)
\n

If the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.

\n

You may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract’s “constructor” (that is, the contract object’s constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a ret variable here Stored in the state database as the actual code of the contract

\n

call contract

The EVM object has three methods to implement the call of the contract, they are:

\n
    \n
  • EVM. Call
    • \n
  • EVM. CallCode
    • \n
  • EVM. DelegateCall
    • \n
  • EVM.StaticCall
    The basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods
    • \n
\n

EVM.CallCode & EVM.DelegateCall

The existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the “library” of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this “library contract”. But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the “library contract”, these properties must be the properties of the “library contract” itself, but this may not be what we want

\n

as an example

\n
1
A -> contractB - delegateCall -> libC
\n

EVM.DelegateCall sets the caller (msg.sender) of the “library contract” (libC) to A, rather than contractB; sets the address of the “library contract” (libC) to contractB.
EVM.CallCode is similar to EVM.DelegateCall. the only difference is that EVM.CallCode only change the address of the “library contract” (libC) to contractB, without chanding the caller to A.
EVM.StaticCall is similar to EVM.Call, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data

\n

during contract call, it first check whether it is precompiled contract. some precompiled contracts are

\n
    \n
  • common.BytesToAddress([]byte{1}): &ecrecover{},
    • \n
  • common.BytesToAddress([]byte{2}): &sha256hash{},
    • \n
  • common.BytesToAddress([]byte{3}): &ripemd160hash{},
    • \n
  • common.BytesToAddress([]byte{4}): &dataCopy{},
    • \n
\n

EVMInterpreter

The interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.

\n

execution layout

\"layout\"

\n

intrinsic gas

The intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.

\n

gas cost

the gas cost of each instruction is stored in JumpTable.operation.dynamicGas or JumpTable.operation.constantGas. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.

\n

In fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.

\n

a method memoryGasCostis used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.

\n

JumpTable

jumptable is 256 sized array of operation

\n

jump instruction

Among the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST

\n
1
2
3
4
5
6
7
8
9
10
11
func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
    pos := stack.pop()
    if !contract.validJumpdest(pos) {
        nop := contract.GetOp(pos.Uint64())
        return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos)
    }
    *pc = pos.Uint64()

    interpreter.intPool.put(pos)
    return nil, nil
}
\n

A function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.

\n

To judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.

\n

Let’s introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the “bit” corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the “bit” of the offset value of the jump destination in this bit vector object is 0

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

overall

the code is under path core/vm
overview of the whole evm module \"evm\"

\n

the core is EVM struct (in evm.go), with main function in creating or call contract. a new EVM object is created every time when processing a transaction. inside the EVM struct, the main items are Interpreter, and StateDB (for state persistence). Interpreter loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in JumpTable (256 sized array of operation)

\n

depending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.

\n

evm

The EVM object is the most important object exported by the evm module, which represents an Ethereum virtual machine

\n

creating evm

Every time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function ApplyTransaction (core/state_processor.go)

\n

creating contract

If the to of the transaction is empty, it means that this transaction is to create a contract, so call EVM.Create to perform related functions

\n
    \n
  • CREATE
    1
    contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))
    • \n
  • CREATE2
    1
    2
    codeAndHash := &codeAndHash{code: code}
    \tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())
    \nduring create contract, an object Contract is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the jumpdests record of the code.
    • \n
\n

then, it invokes below method to create contract

\n
1
2
ret, err := evm.interpreter.Run(contract, nil, false)
evm.StateDB.SetCode(address, ret)
\n

If the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.

\n

You may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract’s “constructor” (that is, the contract object’s constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a ret variable here Stored in the state database as the actual code of the contract

\n

call contract

The EVM object has three methods to implement the call of the contract, they are:

\n
    \n
  • EVM. Call
    • \n
  • EVM. CallCode
    • \n
  • EVM. DelegateCall
    • \n
  • EVM.StaticCall
    The basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods
    • \n
\n

EVM.CallCode & EVM.DelegateCall

The existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the “library” of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this “library contract”. But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the “library contract”, these properties must be the properties of the “library contract” itself, but this may not be what we want

\n

as an example

\n
1
A -> contractB - delegateCall -> libC
\n

EVM.DelegateCall sets the caller (msg.sender) of the “library contract” (libC) to A, rather than contractB; sets the address of the “library contract” (libC) to contractB.
EVM.CallCode is similar to EVM.DelegateCall. the only difference is that EVM.CallCode only change the address of the “library contract” (libC) to contractB, without chanding the caller to A.
EVM.StaticCall is similar to EVM.Call, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data

\n

during contract call, it first check whether it is precompiled contract. some precompiled contracts are

\n
    \n
  • common.BytesToAddress([]byte{1}): &ecrecover{},
    • \n
  • common.BytesToAddress([]byte{2}): &sha256hash{},
    • \n
  • common.BytesToAddress([]byte{3}): &ripemd160hash{},
    • \n
  • common.BytesToAddress([]byte{4}): &dataCopy{},
    • \n
\n

EVMInterpreter

The interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.

\n

execution layout

\"layout\"

\n

intrinsic gas

The intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.

\n

gas cost

the gas cost of each instruction is stored in JumpTable.operation.dynamicGas or JumpTable.operation.constantGas. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.

\n

In fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.

\n

a method memoryGasCostis used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.

\n

JumpTable

jumptable is 256 sized array of operation

\n

jump instruction

Among the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST

\n
1
2
3
4
5
6
7
8
9
10
11
func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
    pos := stack.pop()
    if !contract.validJumpdest(pos) {
        nop := contract.GetOp(pos.Uint64())
        return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos)
    }
    *pc = pos.Uint64()

    interpreter.intPool.put(pos)
    return nil, nil
}
\n

A function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.

\n

To judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.

\n

Let’s introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the “bit” corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the “bit” of the offset value of the jump destination in this bit vector object is 0

\n

references

\n"},{"title":"rust std data structure (2D)","date":"2023-05-02T14:04:38.000Z","_content":"\n## collections\n### BTreeMap\n- `clear(&mut self)` Clears the map, removing all elements.\n- `get(&self, key: &Q)` Returns a reference to the value corresponding to the key.\n- `get_key_value(&self, k: &Q)`\n- `first_key_value(&self)` eturns the first key-value pair in the map.\n- `first_entry(&mut self)` Returns the first entry in the map for in-place manipulation\n- `pop_first(&mut self)` \n- `last_key_value(&self)`\n- `last_entry(&mut self)`\n- `pop_last(&mut self)`\n- `contains_key(&self, key: &Q)`\n- `get_mut(&mut self, key: &Q)` Returns a mutable reference to the value corresponding to the key\n- `insert(&mut self, key: K, value: V)`\n- `try_insert(&mut self, key: K, value: V)` If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.\n- `remove(&mut self, key: &Q)`\n- `remove_entry(&mut self, key: &Q)`\n- `retain(&mut self, mut f: F)` Retains only the elements specified by the predicate.\n```rust\nuse std::collections::BTreeMap;\nlet mut map: BTreeMap = (0..8).map(|x| (x, x*10)).collect();\n// Keep only the elements with even-numbered keys.\nmap.retain(|&k, _| k % 2 == 0);\nassert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));\n```\n- `append(&mut self, other: &mut Self)` Moves all elements from `other` into `self`, leaving `other` empty.\n- `range(&self, range: R) -> Range<'_, K, V>` Constructs a double-ended iterator over a sub-range of elements in the map.\n```rust\nuse std::collections::BTreeMap;\nuse std::ops::Bound::Included;\nlet mut map = BTreeMap::new();\nmap.insert(3, \"a\");\nmap.insert(5, \"b\");\nmap.insert(8, \"c\");\nfor (&key, &value) in map.range((Included(&4), Included(&8))) {\n println!(\"{key}: {value}\");\n}\nassert_eq!(Some((&5, &\"b\")), map.range(4..).next());\n```\n- `range_mut(&mut self, range: R) -> RangeMut<'_, K, V>` \n- `entry(&mut self, key: K)` Gets the given key's corresponding entry in the map for in-place manipulation.\n```rust\nuse std::collections::BTreeMap;\nlet mut count: BTreeMap<&str, usize> = BTreeMap::new();\n// count the number of occurrences of letters in the vec\nfor x in [\"a\", \"b\", \"a\", \"c\", \"a\", \"b\"] {\n count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);\n}\nassert_eq!(count[\"a\"], 3);\nassert_eq!(count[\"b\"], 2);\nassert_eq!(count[\"c\"], 1);\n```\n- `split_off(&mut self, key: &Q)` Splits the collection into two at the given key. Returns everything after the given key,\n- `drain_filter(&mut self, pred: F)` Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns `true`, the element is removed from the map and yielded. If the closure returns `false`, or panics, the element remains in the map and will not be yielded\n- `into_keys(self)` Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this\n- `into_values(self)`\n","source":"_posts/rust/rust_std/rust-std-data-structure-2.md","raw":"---\ntitle: rust std data structure (2D)\ndate: 2023-05-02 22:04:38\ntags: [rust-std]\n---\n\n## collections\n### BTreeMap\n- `clear(&mut self)` Clears the map, removing all elements.\n- `get(&self, key: &Q)` Returns a reference to the value corresponding to the key.\n- `get_key_value(&self, k: &Q)`\n- `first_key_value(&self)` eturns the first key-value pair in the map.\n- `first_entry(&mut self)` Returns the first entry in the map for in-place manipulation\n- `pop_first(&mut self)` \n- `last_key_value(&self)`\n- `last_entry(&mut self)`\n- `pop_last(&mut self)`\n- `contains_key(&self, key: &Q)`\n- `get_mut(&mut self, key: &Q)` Returns a mutable reference to the value corresponding to the key\n- `insert(&mut self, key: K, value: V)`\n- `try_insert(&mut self, key: K, value: V)` If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.\n- `remove(&mut self, key: &Q)`\n- `remove_entry(&mut self, key: &Q)`\n- `retain(&mut self, mut f: F)` Retains only the elements specified by the predicate.\n```rust\nuse std::collections::BTreeMap;\nlet mut map: BTreeMap = (0..8).map(|x| (x, x*10)).collect();\n// Keep only the elements with even-numbered keys.\nmap.retain(|&k, _| k % 2 == 0);\nassert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));\n```\n- `append(&mut self, other: &mut Self)` Moves all elements from `other` into `self`, leaving `other` empty.\n- `range(&self, range: R) -> Range<'_, K, V>` Constructs a double-ended iterator over a sub-range of elements in the map.\n```rust\nuse std::collections::BTreeMap;\nuse std::ops::Bound::Included;\nlet mut map = BTreeMap::new();\nmap.insert(3, \"a\");\nmap.insert(5, \"b\");\nmap.insert(8, \"c\");\nfor (&key, &value) in map.range((Included(&4), Included(&8))) {\n println!(\"{key}: {value}\");\n}\nassert_eq!(Some((&5, &\"b\")), map.range(4..).next());\n```\n- `range_mut(&mut self, range: R) -> RangeMut<'_, K, V>` \n- `entry(&mut self, key: K)` Gets the given key's corresponding entry in the map for in-place manipulation.\n```rust\nuse std::collections::BTreeMap;\nlet mut count: BTreeMap<&str, usize> = BTreeMap::new();\n// count the number of occurrences of letters in the vec\nfor x in [\"a\", \"b\", \"a\", \"c\", \"a\", \"b\"] {\n count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);\n}\nassert_eq!(count[\"a\"], 3);\nassert_eq!(count[\"b\"], 2);\nassert_eq!(count[\"c\"], 1);\n```\n- `split_off(&mut self, key: &Q)` Splits the collection into two at the given key. Returns everything after the given key,\n- `drain_filter(&mut self, pred: F)` Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns `true`, the element is removed from the map and yielded. If the closure returns `false`, or panics, the element remains in the map and will not be yielded\n- `into_keys(self)` Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this\n- `into_values(self)`\n","slug":"rust/rust_std/rust-std-data-structure-2","published":1,"updated":"2023-07-05T13:41:15.719Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk1e4wq3002p6hsj2lwf9hko","content":"

collections

BTreeMap

    \n
  • clear(&mut self) Clears the map, removing all elements.
    • \n
  • get(&self, key: &Q) Returns a reference to the value corresponding to the key.
    • \n
  • get_key_value(&self, k: &Q)
    • \n
  • first_key_value(&self) eturns the first key-value pair in the map.
    • \n
  • first_entry(&mut self) Returns the first entry in the map for in-place manipulation
    • \n
  • pop_first(&mut self)
    • \n
  • last_key_value(&self)
    • \n
  • last_entry(&mut self)
    • \n
  • pop_last(&mut self)
    • \n
  • contains_key(&self, key: &Q)
    • \n
  • get_mut(&mut self, key: &Q) Returns a mutable reference to the value corresponding to the key
    • \n
  • insert(&mut self, key: K, value: V)
    • \n
  • try_insert(&mut self, key: K, value: V) If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.
    • \n
  • remove(&mut self, key: &Q)
    • \n
  • remove_entry(&mut self, key: &Q)
    • \n
  • retain<F>(&mut self, mut f: F) Retains only the elements specified by the predicate.
    1
    2
    3
    4
    5
    use std::collections::BTreeMap;
    let mut map: BTreeMap<i32, i32> = (0..8).map(|x| (x, x*10)).collect();
    // Keep only the elements with even-numbered keys.
    map.retain(|&k, _| k % 2 == 0);
    assert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));
    • \n
  • append(&mut self, other: &mut Self) Moves all elements from other into self, leaving other empty.
    • \n
  • range<T: ?Sized, R>(&self, range: R) -> Range<'_, K, V> Constructs a double-ended iterator over a sub-range of elements in the map.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::collections::BTreeMap;
    use std::ops::Bound::Included;
    let mut map = BTreeMap::new();
    map.insert(3, "a");
    map.insert(5, "b");
    map.insert(8, "c");
    for (&key, &value) in map.range((Included(&4), Included(&8))) {
        println!("{key}: {value}");
    }
    assert_eq!(Some((&5, &"b")), map.range(4..).next());
    • \n
  • range_mut<T: ?Sized, R>(&mut self, range: R) -> RangeMut<'_, K, V>
    • \n
  • entry(&mut self, key: K) Gets the given key’s corresponding entry in the map for in-place manipulation.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    use std::collections::BTreeMap;
    let mut count: BTreeMap<&str, usize> = BTreeMap::new();
    // count the number of occurrences of letters in the vec
    for x in ["a", "b", "a", "c", "a", "b"] {
        count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);
    }
    assert_eq!(count["a"], 3);
    assert_eq!(count["b"], 2);
    assert_eq!(count["c"], 1);
    • \n
  • split_off<Q: ?Sized + Ord>(&mut self, key: &Q) Splits the collection into two at the given key. Returns everything after the given key,
    • \n
  • drain_filter<F>(&mut self, pred: F) Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns true, the element is removed from the map and yielded. If the closure returns false, or panics, the element remains in the map and will not be yielded
    • \n
  • into_keys(self) Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this
    • \n
  • into_values(self)
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

collections

BTreeMap

    \n
  • clear(&mut self) Clears the map, removing all elements.
    • \n
  • get(&self, key: &Q) Returns a reference to the value corresponding to the key.
    • \n
  • get_key_value(&self, k: &Q)
    • \n
  • first_key_value(&self) eturns the first key-value pair in the map.
    • \n
  • first_entry(&mut self) Returns the first entry in the map for in-place manipulation
    • \n
  • pop_first(&mut self)
    • \n
  • last_key_value(&self)
    • \n
  • last_entry(&mut self)
    • \n
  • pop_last(&mut self)
    • \n
  • contains_key(&self, key: &Q)
    • \n
  • get_mut(&mut self, key: &Q) Returns a mutable reference to the value corresponding to the key
    • \n
  • insert(&mut self, key: K, value: V)
    • \n
  • try_insert(&mut self, key: K, value: V) If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.
    • \n
  • remove(&mut self, key: &Q)
    • \n
  • remove_entry(&mut self, key: &Q)
    • \n
  • retain<F>(&mut self, mut f: F) Retains only the elements specified by the predicate.
    1
    2
    3
    4
    5
    use std::collections::BTreeMap;
    let mut map: BTreeMap<i32, i32> = (0..8).map(|x| (x, x*10)).collect();
    // Keep only the elements with even-numbered keys.
    map.retain(|&k, _| k % 2 == 0);
    assert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));
    • \n
  • append(&mut self, other: &mut Self) Moves all elements from other into self, leaving other empty.
    • \n
  • range<T: ?Sized, R>(&self, range: R) -> Range<'_, K, V> Constructs a double-ended iterator over a sub-range of elements in the map.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::collections::BTreeMap;
    use std::ops::Bound::Included;
    let mut map = BTreeMap::new();
    map.insert(3, "a");
    map.insert(5, "b");
    map.insert(8, "c");
    for (&key, &value) in map.range((Included(&4), Included(&8))) {
        println!("{key}: {value}");
    }
    assert_eq!(Some((&5, &"b")), map.range(4..).next());
    • \n
  • range_mut<T: ?Sized, R>(&mut self, range: R) -> RangeMut<'_, K, V>
    • \n
  • entry(&mut self, key: K) Gets the given key’s corresponding entry in the map for in-place manipulation.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    use std::collections::BTreeMap;
    let mut count: BTreeMap<&str, usize> = BTreeMap::new();
    // count the number of occurrences of letters in the vec
    for x in ["a", "b", "a", "c", "a", "b"] {
        count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);
    }
    assert_eq!(count["a"], 3);
    assert_eq!(count["b"], 2);
    assert_eq!(count["c"], 1);
    • \n
  • split_off<Q: ?Sized + Ord>(&mut self, key: &Q) Splits the collection into two at the given key. Returns everything after the given key,
    • \n
  • drain_filter<F>(&mut self, pred: F) Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns true, the element is removed from the map and yielded. If the closure returns false, or panics, the element remains in the map and will not be yielded
    • \n
  • into_keys(self) Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this
    • \n
  • into_values(self)
    • \n
\n"}],"PostAsset":[],"PostCategory":[],"PostTag":[{"post_id":"clk1e4wpi00006hsj5wv8brh1","tag_id":"clk1e4wpm00026hsj17rx1ohz","_id":"clk1e4wpq000b6hsjb0v41axy"},{"post_id":"clk1e4wpi00006hsj5wv8brh1","tag_id":"clk1e4wpp00066hsj8grh49w0","_id":"clk1e4wpr000d6hsj4e3s1aoi"},{"post_id":"clk1e4wpl00016hsjdgw02i4v","tag_id":"clk1e4wpm00026hsj17rx1ohz","_id":"clk1e4wpr000g6hsj1yp6e4ck"},{"post_id":"clk1e4wpp00076hsjh7rifext","tag_id":"clk1e4wpr000e6hsj1d9v3dyn","_id":"clk1e4wps000k6hsj3qjt9c2w"},{"post_id":"clk1e4wpp00086hsjfqu01qqy","tag_id":"clk1e4wpr000e6hsj1d9v3dyn","_id":"clk1e4wps000o6hsjhspx4hlj"},{"post_id":"clk1e4wps000n6hsj472tg11o","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpt000q6hsj01v034w8"},{"post_id":"clk1e4wpq000a6hsj4hyvg73a","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpt000t6hsjey7ycy58"},{"post_id":"clk1e4wps000p6hsj78b40thl","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpu000v6hsjf4g427gd"},{"post_id":"clk1e4wpt000s6hsjgtjq4nac","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpu000y6hsjdj4612by"},{"post_id":"clk1e4wpq000c6hsj11cxhldu","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpu00106hsj8ru78bct"},{"post_id":"clk1e4wpt000u6hsj6fqz3nu7","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpv00136hsj9jll9d3j"},{"post_id":"clk1e4wpu000x6hsj51636y2x","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpv00156hsj0afygljr"},{"post_id":"clk1e4wpr000f6hsj2rgy3f9c","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpv00186hsjhz3weksu"},{"post_id":"clk1e4wpu000z6hsjb7jy64m2","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpv001a6hsj2h6l6lka"},{"post_id":"clk1e4wpr000h6hsj2ag6e8t0","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpw001d6hsj9oq392wr"},{"post_id":"clk1e4wpv00146hsj0upzhkpn","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpw001f6hsj9jebdotb"},{"post_id":"clk1e4wpv00176hsjd2sh9yph","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpw001i6hsj9vp64kkr"},{"post_id":"clk1e4wps000j6hsj1nk1429h","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpw001k6hsj4a478cfj"},{"post_id":"clk1e4wpv00196hsjcp72dtgr","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpx001n6hsj4rze196n"},{"post_id":"clk1e4wps000l6hsja9odhbdf","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpy001p6hsjfayjfekz"},{"post_id":"clk1e4wpu00126hsjf6vr6ylq","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wpz001s6hsj7ceu0ifp"},{"post_id":"clk1e4wpu00126hsjf6vr6ylq","tag_id":"clk1e4wpw001g6hsja3qibdw0","_id":"clk1e4wpz001t6hsj6c2tct31"},{"post_id":"clk1e4wpx001o6hsj5m9db7qe","tag_id":"clk1e4wpp00066hsj8grh49w0","_id":"clk1e4wpz001v6hsj8aqfbvpv"},{"post_id":"clk1e4wpv001c6hsjfftj9tr6","tag_id":"clk1e4wpw001m6hsjgy5j94y8","_id":"clk1e4wpz001w6hsjduo998rh"},{"post_id":"clk1e4wpy001q6hsj0ohw8qa1","tag_id":"clk1e4wpp00066hsj8grh49w0","_id":"clk1e4wpz001y6hsjfhxqdn2i"},{"post_id":"clk1e4wpw001e6hsj1lfmhvaa","tag_id":"clk1e4wpw001m6hsjgy5j94y8","_id":"clk1e4wpz001z6hsjbl4t0enl"},{"post_id":"clk1e4wpw001h6hsj7tdyclkv","tag_id":"clk1e4wpw001m6hsjgy5j94y8","_id":"clk1e4wpz00216hsj6r2d62im"},{"post_id":"clk1e4wpw001j6hsj9vpw0y0r","tag_id":"clk1e4wpw001m6hsjgy5j94y8","_id":"clk1e4wpz00226hsj3x8y952v"},{"post_id":"clk1e4wpw001l6hsjedcxczok","tag_id":"clk1e4wpw001m6hsjgy5j94y8","_id":"clk1e4wpz00246hsj77r4glp0"},{"post_id":"clk1e4wpw001l6hsjedcxczok","tag_id":"clk1e4wpz00236hsjgvue65w3","_id":"clk1e4wpz00256hsj24j0fbys"},{"post_id":"clk1e4wq000266hsjd3g4hlvb","tag_id":"clk1e4wps000m6hsj1p6yf3ai","_id":"clk1e4wq000286hsjdjm4h9gj"},{"post_id":"clk1e4wq1002d6hsjg0dbh2fw","tag_id":"clk1e4wpm00026hsj17rx1ohz","_id":"clk1e4wq2002g6hsj3pvr2oh8"},{"post_id":"clk1e4wq1002d6hsjg0dbh2fw","tag_id":"clk1e4wpp00066hsj8grh49w0","_id":"clk1e4wq2002j6hsjc0f7h525"},{"post_id":"clk1e4wq2002f6hsj4pqoevwn","tag_id":"clk1e4wpp00066hsj8grh49w0","_id":"clk1e4wq2002l6hsj4w9e5eg1"},{"post_id":"clk1e4wq000276hsjgnt73tin","tag_id":"clk1e4wpw001m6hsjgy5j94y8","_id":"clk1e4wq3002o6hsjfcr822b0"},{"post_id":"clk1e4wq000276hsjgnt73tin","tag_id":"clk1e4wq1002a6hsj6rrjc23x","_id":"clk1e4wq3002q6hsjc3io5cz1"},{"post_id":"clk1e4wq000276hsjgnt73tin","tag_id":"clk1e4wq2002e6hsj07ygagdz","_id":"clk1e4wq3002r6hsj6zil7ayk"},{"post_id":"clk1e4wq2002k6hsjamouhgu0","tag_id":"clk1e4wpm00026hsj17rx1ohz","_id":"clk1e4wq3002t6hsjdczogxhl"},{"post_id":"clk1e4wq2002k6hsjamouhgu0","tag_id":"clk1e4wpp00066hsj8grh49w0","_id":"clk1e4wq3002u6hsj5whxbbt6"},{"post_id":"clk1e4wq000296hsjhr1y1ijp","tag_id":"clk1e4wq2002i6hsj8ybgdjep","_id":"clk1e4wq3002w6hsj8er3giz8"},{"post_id":"clk1e4wq2002m6hsjg7yx2331","tag_id":"clk1e4wpm00026hsj17rx1ohz","_id":"clk1e4wq3002x6hsjdh5wcx19"},{"post_id":"clk1e4wq2002m6hsjg7yx2331","tag_id":"clk1e4wpp00066hsj8grh49w0","_id":"clk1e4wq3002y6hsj7je7hxfh"},{"post_id":"clk1e4wq3002p6hsj2lwf9hko","tag_id":"clk1e4wq2002n6hsjaw1a19xt","_id":"clk1e4wq3002z6hsj4ltyg43g"},{"post_id":"clk1e4wq1002b6hsj2xfef22t","tag_id":"clk1e4wq2002n6hsjaw1a19xt","_id":"clk1e4wq400306hsj07vr83ia"},{"post_id":"clk1e4wq1002c6hsj7olz9y4o","tag_id":"clk1e4wq2002n6hsjaw1a19xt","_id":"clk1e4wq400316hsj8dd7gu8h"},{"post_id":"clk1e4wq2002h6hsjd2u10w1i","tag_id":"clk1e4wq2002n6hsjaw1a19xt","_id":"clk1e4wq400326hsj5ljo11og"}],"Tag":[{"name":"blockchain","_id":"clk1e4wpm00026hsj17rx1ohz"},{"name":"geth","_id":"clk1e4wpp00066hsj8grh49w0"},{"name":"golang","_id":"clk1e4wpr000e6hsj1d9v3dyn"},{"name":"rust","_id":"clk1e4wps000m6hsj1p6yf3ai"},{"name":"cargo","_id":"clk1e4wpw001g6hsja3qibdw0"},{"name":"cryptography","_id":"clk1e4wpw001m6hsjgy5j94y8"},{"name":"zkp","_id":"clk1e4wpz00236hsjgvue65w3"},{"name":"mpc","_id":"clk1e4wq1002a6hsj6rrjc23x"},{"name":"ecdsa","_id":"clk1e4wq2002e6hsj07ygagdz"},{"name":"rust-crate","_id":"clk1e4wq2002i6hsj8ybgdjep"},{"name":"rust-std","_id":"clk1e4wq2002n6hsjaw1a19xt"}]}} \ No newline at end of file +{"meta":{"version":1,"warehouse":"4.0.2"},"models":{"Asset":[{"_id":"node_modules/hexo-theme-landscape/source/css/style.styl","path":"css/style.styl","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.css","path":"fancybox/jquery.fancybox.min.css","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.js","path":"fancybox/jquery.fancybox.min.js","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/js/jquery-3.4.1.min.js","path":"js/jquery-3.4.1.min.js","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/js/script.js","path":"js/script.js","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/FontAwesome.otf","path":"css/fonts/FontAwesome.otf","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.eot","path":"css/fonts/fontawesome-webfont.eot","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.svg","path":"css/fonts/fontawesome-webfont.svg","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.ttf","path":"css/fonts/fontawesome-webfont.ttf","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff","path":"css/fonts/fontawesome-webfont.woff","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff2","path":"css/fonts/fontawesome-webfont.woff2","modified":1,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/images/banner.jpg","path":"css/images/banner.jpg","modified":1,"renderable":1},{"_id":"source/2017-552.pdf","path":"2017-552.pdf","modified":1,"renderable":0},{"_id":"source/images/evm.layout.png","path":"images/evm.layout.png","modified":1,"renderable":0},{"_id":"source/images/evm.drawio.google.png","path":"images/evm.drawio.google.png","modified":1,"renderable":0},{"_id":"source/images/kademlia.onlineprob.png","path":"images/kademlia.onlineprob.png","modified":1,"renderable":0},{"_id":"source/images/geth_starts.drawio.png","path":"images/geth_starts.drawio.png","modified":1,"renderable":0},{"_id":"source/images/kademlia.locating.png","path":"images/kademlia.locating.png","modified":1,"renderable":0},{"_id":"source/images/kademlia.subtree.png","path":"images/kademlia.subtree.png","modified":1,"renderable":0},{"_id":"source/images/mpt.state.ref.png","path":"images/mpt.state.ref.png","modified":1,"renderable":0},{"_id":"source/images/mpt.png","path":"images/mpt.png","modified":1,"renderable":0},{"_id":"source/images/trie.prefix.png","path":"images/trie.prefix.png","modified":1,"renderable":0},{"_id":"source/images/geth/sync.mode.jpg","path":"images/geth/sync.mode.jpg","modified":1,"renderable":0},{"_id":"source/images/paillier/carmichael_thorem.png","path":"images/paillier/carmichael_thorem.png","modified":1,"renderable":0},{"_id":"source/images/paillier/carmichael_thorem_2.png","path":"images/paillier/carmichael_thorem_2.png","modified":1,"renderable":0},{"_id":"source/images/paillier/homomorphic_addition.png","path":"images/paillier/homomorphic_addition.png","modified":1,"renderable":0},{"_id":"source/images/paillier/homomorphic_mul.png","path":"images/paillier/homomorphic_mul.png","modified":1,"renderable":0},{"_id":"source/images/two_party_ecdsa/paillier_enc.png","path":"images/two_party_ecdsa/paillier_enc.png","modified":1,"renderable":0},{"_id":"source/images/two_party_ecdsa/schnorr_ecdsa_comparison.png","path":"images/two_party_ecdsa/schnorr_ecdsa_comparison.png","modified":1,"renderable":0},{"_id":"source/images/cryptography/rsa/rsa_signature.png","path":"images/cryptography/rsa/rsa_signature.png","modified":1,"renderable":0},{"_id":"source/images/cryptography/elliptic_curve/point_addition.webp","path":"images/cryptography/elliptic_curve/point_addition.webp","modified":1,"renderable":0},{"_id":"source/images/rust/macros/16.compile_process.png","path":"images/rust/macros/16.compile_process.png","modified":1,"renderable":0},{"_id":"source/images/rust/ownership/move-string-2.png","path":"images/rust/ownership/move-string-2.png","modified":1,"renderable":0},{"_id":"source/images/rust/ownership/move-string.png","path":"images/rust/ownership/move-string.png","modified":1,"renderable":0},{"_id":"source/images/rust/memory/trait_object_memory.png","path":"images/rust/memory/trait_object_memory.png","modified":1,"renderable":0},{"_id":"source/images/rust/pointers/cycle.ref.png","path":"images/rust/pointers/cycle.ref.png","modified":1,"renderable":0},{"_id":"source/images/rust/pointers/rc.png","path":"images/rust/pointers/rc.png","modified":1,"renderable":0}],"Cache":[{"_id":"node_modules/hexo-theme-landscape/package.json","hash":"9a94875cbf4c27fbe2e63da0496242addc6d2876","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/_config.yml","hash":"b608c1f1322760dce9805285a602a95832730a2e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/LICENSE","hash":"c480fce396b23997ee23cc535518ffaaf7f458f8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/README.md","hash":"d2772ece6d4422ccdaa0359c3e07588834044052","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/de.yml","hash":"3ebf0775abbee928c8d7bda943c191d166ded0d3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/en.yml","hash":"3083f319b352d21d80fc5e20113ddf27889c9d11","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/es.yml","hash":"76edb1171b86532ef12cfd15f5f2c1ac3949f061","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ja.yml","hash":"a73e1b9c80fd6e930e2628b393bfe3fb716a21a9","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/hu.yml","hash":"284d557130bf54a74e7dcef9d42096130e4d9550","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/fr.yml","hash":"415e1c580ced8e4ce20b3b0aeedc3610341c76fb","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/mn.yml","hash":"2e7523951072a9403ead3840ad823edd1084c116","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ko.yml","hash":"881d6a0a101706e0452af81c580218e0bfddd9cf","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/nl.yml","hash":"12ed59faba1fc4e8cdd1d42ab55ef518dde8039c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/pt.yml","hash":"57d07b75d434fbfc33b0ddb543021cb5f53318a8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/no.yml","hash":"965a171e70347215ec726952e63f5b47930931ef","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/tr.yml","hash":"a1cdbfa17682d7a971de8ab8588bf57c74224b5b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/it.yml","hash":"89b7d91306b2c1a0f3ac023b657bf974f798a1e8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/zh-CN.yml","hash":"1efd95774f401c80193eac6ee3f1794bfe93dc5a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/page.ejs","hash":"7d80e4e36b14d30a7cd2ac1f61376d9ebf264e8b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/zh-TW.yml","hash":"53ce3000c5f767759c7d2c4efcaa9049788599c3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ru.yml","hash":"4fda301bbd8b39f2c714e2c934eccc4b27c0a2b0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/post.ejs","hash":"7d80e4e36b14d30a7cd2ac1f61376d9ebf264e8b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/index.ejs","hash":"aa1b4456907bdb43e629be3931547e2d29ac58c8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/archive.ejs","hash":"2703b07cc8ac64ae46d1d263f4653013c7e1666b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/tag.ejs","hash":"eaa7b4ccb2ca7befb90142e4e68995fb1ea68b2e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/scripts/fancybox.js","hash":"c857d7a5e4a5d71c743a009c5932bf84229db428","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/category.ejs","hash":"765426a9c8236828dc34759e604cc2c52292835a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/layout.ejs","hash":"0d1765036e4874500e68256fedb7470e96eeb6ee","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/category.ejs","hash":"dd1e5af3c6af3f5d6c85dfd5ca1766faed6a0b05","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/tag.ejs","hash":"2de380865df9ab5f577f7d3bcadf44261eb5faae","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/recent_posts.ejs","hash":"60c4b012dcc656438ff59997e60367e5a21ab746","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/archive.ejs","hash":"beb4a86fcc82a9bdda9289b59db5a1988918bec3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/tagcloud.ejs","hash":"b4a2079101643f63993dcdb32925c9b071763b46","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/after-footer.ejs","hash":"414914ebb159fac1922b056b905e570ac7521925","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/archive.ejs","hash":"7cb70a7a54f8c7ae49b10d1f37c0a9b74eab8826","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/footer.ejs","hash":"3656eb692254346671abc03cb3ba1459829e0dce","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/article.ejs","hash":"dfd555c00e85ffc4207c88968d12b219c1f086ec","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/archive-post.ejs","hash":"c7a71425a946d05414c069ec91811b5c09a92c47","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/gauges-analytics.ejs","hash":"21a1e2a3907d1a3dad1cd0ab855fe6735f233c74","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/google-analytics.ejs","hash":"2ea7442ea1e1a8ab4e41e26c563f58413b59a3d0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/head.ejs","hash":"f215d92a882247a7cc5ea80b241bedfcec0ea6ca","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/header.ejs","hash":"c1acd247e14588cdf101a69460cb8319c18cd078","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/sidebar.ejs","hash":"930da35cc2d447a92e5ee8f835735e6fd2232469","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_variables.styl","hash":"581b0cbefdaa5f894922133989dd2d3bf71ded79","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/style.styl","hash":"9c451e5efd72c5bb8b56e8c2b94be731e99db05b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/mobile-nav.ejs","hash":"e952a532dfc583930a666b9d4479c32d4a84b44e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_extend.styl","hash":"222fbe6d222531d61c1ef0f868c90f747b1c2ced","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/js/script.js","hash":"998ed4c5b147e1299bf62beebf33514474f28112","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.css","hash":"1be9b79be02a1cfc5d96c4a5e0feb8f472babd95","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/category.ejs","hash":"c6bcd0e04271ffca81da25bcff5adf3d46f02fc0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/date.ejs","hash":"f1458584b679545830b75bef2526e2f3eb931045","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/gallery.ejs","hash":"3d9d81a3c693ff2378ef06ddb6810254e509de5b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/nav.ejs","hash":"16a904de7bceccbb36b4267565f2215704db2880","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/title.ejs","hash":"4d7e62574ddf46de9b41605fe3140d77b5ddb26d","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/tag.ejs","hash":"2fcb0bf9c8847a644167a27824c9bb19ac74dd14","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/article.styl","hash":"80759482d07063c091e940f964a1cf6693d3d406","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/archive.styl","hash":"db15f5677dc68f1730e82190bab69c24611ca292","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/footer.styl","hash":"e35a060b8512031048919709a8e7b1ec0e40bc1b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/comment.styl","hash":"79d280d8d203abb3bd933ca9b8e38c78ec684987","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/highlight.styl","hash":"bf4e7be1968dad495b04e83c95eac14c4d0ad7c0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/header.styl","hash":"85ab11e082f4dd86dde72bed653d57ec5381f30c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar-aside.styl","hash":"890349df5145abf46ce7712010c89237900b3713","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/mobile.styl","hash":"a399cf9e1e1cec3e4269066e2948d7ae5854d745","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar-bottom.styl","hash":"8fd4f30d319542babfd31f087ddbac550f000a8a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar.styl","hash":"404ec059dc674a48b9ab89cd83f258dec4dcb24d","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_util/grid.styl","hash":"0bf55ee5d09f193e249083602ac5fcdb1e571aed","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_util/mixin.styl","hash":"44f32767d9fd3c1c08a60d91f181ee53c8f0dbb3","modified":1669606834570},{"_id":"source/_posts/MPT.md","hash":"1d0394f11c3361b4474dbe49ced5c5751144fe91","modified":1681534320319},{"_id":"source/_posts/hello-world.md","hash":"8241e671f980a667f875b9a6493f17bd333a1cfb","modified":1680799419107},{"_id":"source/_posts/blockchain.kademlia.md","hash":"0cb0522a114bc53d79f2db4a1bf2a02c29ef1c2f","modified":1681457596860},{"_id":"source/_posts/geth-fine-tune.md","hash":"5431cd654f7152fd5c590891a53568f54eec4125","modified":1682416094119},{"_id":"source/images/evm.layout.png","hash":"6927cb3b922b2bcec926ec5d797e7ea8ea2b5d00","modified":1681440784560},{"_id":"source/images/kademlia.onlineprob.png","hash":"98c55aa819ef7047b3749c89eb4546f0d799a239","modified":1681443973575},{"_id":"source/images/mpt.state.ref.png","hash":"ec747cf092820c0ab5f72d0f4f07f7705bb9ecf0","modified":1681533561017},{"_id":"source/_posts/cryptography/cryptography-01-primitive-group-and-field.md","hash":"b109aef9a3c4ca55a7198c284cd007716b094044","modified":1689264071422},{"_id":"source/_posts/cryptography/cryptography-04-digital-signature.md","hash":"f2539c849c5e052c62123a7fde737ce4c0300134","modified":1689472839727},{"_id":"source/_posts/cryptography/cryptography-02-rsa.md","hash":"6c0bb57a988b036e8b8beca7343b69c025bc4ea7","modified":1689404064042},{"_id":"source/_posts/cryptography/cryptography-03-elliptic-curve.md","hash":"3ee7e6e7b609605f7c26d5bf1f7508771d6c7a95","modified":1689403973426},{"_id":"source/_posts/cryptography/two-party-ecdsa.md","hash":"e94506f2f21233958c8f48184fad671919f32f8b","modified":1682317913595},{"_id":"source/_posts/golang/go-reflect.md","hash":"c2808bbd3bf422ab5679c246444975107b98fb1e","modified":1687070564968},{"_id":"source/_posts/golang/go-similar-concepts-comparison.md","hash":"5de26ef1a5907db4264ff5e491b75aa2dfb43af1","modified":1687072042116},{"_id":"source/_posts/rust/rust-02-basics.md","hash":"be1111d868417c54b8b019938b8144e8be35df1f","modified":1682932033124},{"_id":"source/_posts/rust/rust-03-ownership.md","hash":"7d39498532088f438d76f1d0b42892bc985c0c95","modified":1682947052602},{"_id":"source/_posts/rust/rust-01-resource.md","hash":"5e2c159128ccef35da0d3a2a72b6bb7e1c12fc73","modified":1688011565747},{"_id":"source/_posts/rust/rust-04-lifetime.md","hash":"d338498d7d159a3f94687082a10fc28ada706b16","modified":1682950129387},{"_id":"source/_posts/rust/rust-05-memory-layout.md","hash":"9a8c7dac3a23bca5f7692a3f6c0c8827dfd8c7e5","modified":1683082672719},{"_id":"source/_posts/rust/rust-06-smart-pointer.md","hash":"909ecf0ecf594651191e0953eca17aca7fa64669","modified":1683099103613},{"_id":"source/_posts/rust/rust-08-project-management.md","hash":"2c1ab1e7b8c8510935a694e33aa2c676437f0fd1","modified":1683099708892},{"_id":"source/_posts/rust/rust-async.md","hash":"f8b896f06752fcdb3c9fa34d2ed0ba958aef9c49","modified":1683106393753},{"_id":"source/_posts/rust/rust-07-macro.md","hash":"f6e51943ab413f5462baca1327c53ac20a8afcda","modified":1682950710350},{"_id":"source/_posts/rust/rust-cargo-all-in-one.md","hash":"27eb587fe52f0461e1e30ed82b5d10a806e168f0","modified":1683108258682},{"_id":"source/_posts/rust/rust-cargo-doc.md","hash":"52303be5d9e342444a4cb661fa418ab68b28d640","modified":1683106131353},{"_id":"source/_posts/rust/rust-09-functional.md","hash":"b2e1f9a46e02c5aa49ea19394e074777558a51eb","modified":1688003621688},{"_id":"source/_posts/rust/rust-10-concurrency.md","hash":"5bba6d7c1b0e3a24f07a4899f1162a93af8ad5b1","modified":1688283743897},{"_id":"source/_posts/rust/rust-tools.md","hash":"3019a116f156d05a95fd8fc76fa8b1380f778f24","modified":1683105904042},{"_id":"source/_posts/rust/rust-similar-concepts-comparison.md","hash":"17c7d67efd6315828a09762c633e7f8a0c9cdee2","modified":1688891607383},{"_id":"source/_posts/rust/rust-sugar.md","hash":"dfa5a3f7bfd985118b97ae9055da496778b604e8","modified":1689060987147},{"_id":"source/_posts/cryptography/paillier-encryption.md","hash":"4e43aa2d126bcb334563262563071c764c3ae19f","modified":1682317904177},{"_id":"source/images/geth/sync.mode.jpg","hash":"657cc148abc14f01e7483265c67936e73f79d0e4","modified":1687272599480},{"_id":"source/_posts/cryptography/zkp/zkp-a-brief-understanding.md","hash":"2e0f55f389b56b49dea2418cf062722ca339a0ea","modified":1689173902472},{"_id":"source/images/paillier/homomorphic_addition.png","hash":"26b0e4a070a7e29710e2ceace09bbdb79b3a883e","modified":1682317632123},{"_id":"source/images/paillier/homomorphic_mul.png","hash":"e8a2420501140a8d48db202065aa221df92f19dc","modified":1682317735470},{"_id":"source/_posts/rust/crates/rust-serde.md","hash":"9b985750a751a7bd28effe9e97147e4207a5f093","modified":1688053726930},{"_id":"source/images/two_party_ecdsa/paillier_enc.png","hash":"132cdc74eb588951d1bd347b1b6e825912bc0460","modified":1682232953667},{"_id":"source/_posts/geth/code_analysis/geth.0.get.start.md","hash":"6cd5256bae5e43f3dfcd341e27c52eccf8848f60","modified":1682434784499},{"_id":"source/_posts/geth/code_analysis/geth.1.rpc.md","hash":"623aec4f3cd8afb85b7b30d8bfde50bb2e5a4d4a","modified":1682614464069},{"_id":"source/_posts/rust/crates/rust-frequently-used-crates.md","hash":"39aec8a77f62d6c3b164ecef0663a612423afd63","modified":1683106159269},{"_id":"source/_posts/geth/code_analysis/geth.evm.md","hash":"ecea3e42003911dd58a2d6a9b8293f02ccb16a75","modified":1681441326144},{"_id":"source/_posts/geth/tech_docs/geth.prune.md","hash":"9ab8f315c3374f67ff7ac6f74a7fbef0ca9f7894","modified":1687341103628},{"_id":"source/_posts/rust/rust_std/rust-std-data-structure-1.md","hash":"34627ff9cff48a6a79a36d4e88cc4d32e118c3ae","modified":1689070720048},{"_id":"source/_posts/geth/tech_docs/geth.sync.mode.md","hash":"7502b826b5ecbdd02f759a1780528dae344ccad7","modified":1687309420833},{"_id":"source/_posts/geth/tech_docs/geth.v1.10.0.md","hash":"d303922d31b987d5b57080fb6833b84e568de342","modified":1687100789245},{"_id":"source/_posts/rust/rust_std/rust-std-data-structure-2.md","hash":"32b80b9540433ffa77a3a7969e06921eb9ddbbca","modified":1688564475719},{"_id":"source/_posts/rust/rust_std/rust-smart-pointer-and-internal-mutibility.md","hash":"d97435f2c35ffcd4feb8a1aff787c7c20922438a","modified":1688915715385},{"_id":"source/images/cryptography/elliptic_curve/point_addition.webp","hash":"ba6cd29aec4a694b53933d0824df180158f98316","modified":1689255155658},{"_id":"source/_posts/rust/rust_std/rust-std-sync.md","hash":"bf648427835ab0d834b2701b28bdfd35088aeb2e","modified":1688566866619},{"_id":"source/images/rust/ownership/move-string.png","hash":"d708edd6fb84b9d4ebe75556932f6b920934ca9a","modified":1682934539699},{"_id":"source/images/rust/memory/trait_object_memory.png","hash":"dd750cffa7bca5bde7856cfb65f4500de286c96f","modified":1683082638012},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.js","hash":"6181412e73966696d08e1e5b1243a572d0f22ba6","modified":1669606834570},{"_id":"source/images/rust/pointers/rc.png","hash":"f568f69808ef0357ea5e6d42667d863b1139cbf1","modified":1683085079705},{"_id":"node_modules/hexo-theme-landscape/source/js/jquery-3.4.1.min.js","hash":"88523924351bac0b5d560fe0c5781e2556e7693d","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff","hash":"28b782240b3e76db824e12c02754a9731a167527","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff2","hash":"d6f48cba7d076fb6f2fd6ba993a75b9dc1ecbf0c","modified":1669606834570},{"_id":"source/images/evm.drawio.google.png","hash":"413a44d66153c93e2c31172d988e051a2bed9940","modified":1681436195264},{"_id":"source/images/geth_starts.drawio.png","hash":"3c426bd608121669f232aaa1b05ed7a342287fc8","modified":1682005494018},{"_id":"source/images/kademlia.subtree.png","hash":"d3198467460972151f4b2fe2b56fab0dd9e411ca","modified":1681442854122},{"_id":"source/images/trie.prefix.png","hash":"67d26d416faf0cc43ee35b4acda598e88dad2949","modified":1681520956971},{"_id":"source/images/paillier/carmichael_thorem.png","hash":"75b4d32eb2233db653bca52cdea4a624555b5ce4","modified":1682316092261},{"_id":"source/images/paillier/carmichael_thorem_2.png","hash":"29b103c94c36f4520ca8b675486af3d914998ac1","modified":1682316415073},{"_id":"source/images/two_party_ecdsa/schnorr_ecdsa_comparison.png","hash":"69cbed302af467a4e99653dfb51dca45c4a5a6f3","modified":1682231680569},{"_id":"source/images/rust/ownership/move-string-2.png","hash":"83342aed7ee254c099ada5d49ed8b9ba52a451a0","modified":1682934544128},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/FontAwesome.otf","hash":"048707bc52ac4b6563aaa383bfe8660a0ddc908c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.ttf","hash":"13b1eab65a983c7a73bc7997c479d66943f7c6cb","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.eot","hash":"d980c2ce873dc43af460d4d572d441304499f400","modified":1669606834570},{"_id":"source/images/rust/pointers/cycle.ref.png","hash":"ed99e2c6020833ccd5af751cf6eb2031cf47d9aa","modified":1683098263386},{"_id":"node_modules/hexo-theme-landscape/source/css/images/banner.jpg","hash":"f44aa591089fcb3ec79770a1e102fd3289a7c6a6","modified":1669606834570},{"_id":"source/images/kademlia.locating.png","hash":"702d6b779294a3c6e033cc9bde14ef8950982310","modified":1681455579355},{"_id":"source/images/rust/macros/16.compile_process.png","hash":"444080319ca2101672941346988f78ed9a37c32d","modified":1682908885234},{"_id":"source/images/mpt.png","hash":"700032035bdcd793f94da522330552727b00e5a3","modified":1681533347412},{"_id":"source/images/cryptography/rsa/rsa_signature.png","hash":"44eb1a608dafaae244e487cf082aa79c2af5c19f","modified":1689403998940},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.svg","hash":"98a8aa5cf7d62c2eff5f07ede8d844b874ef06ed","modified":1669606834570},{"_id":"source/2017-552.pdf","hash":"b1a08857c1f6532f1fbb718c48a34fd48ea7da70","modified":1682236639612},{"_id":"public/2023/06/20/cryptography/zkp/zkp-a-brief-understanding/index.html","hash":"a910b96f5d6fa826b39e43b0c9da16db046765f0","modified":1689472919651},{"_id":"public/2023/06/17/cryptography/cryptography-03-elliptic-curve/index.html","hash":"dc7ab203ce90f47e6508ccf0de86ced746e20b6a","modified":1689472919651},{"_id":"public/2023/06/01/rust/rust-10-concurrency/index.html","hash":"c9170c0f3faf9eabcdcb46aab86905b5383169dd","modified":1689472919651},{"_id":"public/2023/04/11/rust/rust-09-functional/index.html","hash":"9047f91c23605b39064f09b70467563bd755d3f3","modified":1689472919651},{"_id":"public/2023/04/01/rust/crates/rust-serde/index.html","hash":"71dc7441dbf9e7884ae23b0229ee189eca42af1e","modified":1689472919651},{"_id":"public/2023/03/25/geth/tech_docs/geth.prune/index.html","hash":"2a569caebd01b2f70780c9c65bbeb2e7dab06b21","modified":1689472919651},{"_id":"public/2023/03/05/golang/go-similar-concepts-comparison/index.html","hash":"51fc2b0c2bfb0015cbb5ddd3f948e5efd7b63c8f","modified":1689472919651},{"_id":"public/2023/01/22/MPT/index.html","hash":"658e02c7ccd2d41d04004cdd4cddc230dba48c3c","modified":1689472919651},{"_id":"public/2023/02/07/cryptography/two-party-ecdsa/index.html","hash":"11b20a49e43285941611ec3ac2de72a222c880a6","modified":1689472919651},{"_id":"public/2023/01/01/geth-fine-tune/index.html","hash":"a3f2a442b498aace48b4b752c1654d113d57527f","modified":1689472919651},{"_id":"public/2022/12/13/rust/crates/rust-frequently-used-crates/index.html","hash":"c52b399cd90ca7ed175cbdb2c2237020fce89b26","modified":1689472919651},{"_id":"public/2022/11/27/hello-world/index.html","hash":"afd11c86871e8f9295168d2abcd8a7caf0d42b3c","modified":1689472919651},{"_id":"public/2022/11/20/rust/rust-tools/index.html","hash":"102062d12c74375c128d40b12fc47f402f8d5c99","modified":1689472919651},{"_id":"public/2022/11/13/rust/rust-cargo-doc/index.html","hash":"9dd99d5db8bef1f10b22f0edc474b1490b4d9bfe","modified":1689472919651},{"_id":"public/2022/11/08/geth/code_analysis/geth.1.rpc/index.html","hash":"b49b4b991131106b1ab811971868d97f7e99ad75","modified":1689472919651},{"_id":"public/2022/10/25/rust/rust-05-memory-layout/index.html","hash":"8b9eee275f76633c65a6fc2093e78ca48274e78e","modified":1689472919651},{"_id":"public/2022/09/27/rust/rust-01-resource/index.html","hash":"61ed6d14cc860aaa65cd01e9d80ffa30c304bea9","modified":1689472919651},{"_id":"public/archives/page/2/index.html","hash":"7af260de532e14dd89e5a30b4a37124c8916b427","modified":1689472919651},{"_id":"public/archives/index.html","hash":"aa1487875e2f64aa5dc7ec75fbe65082f21a6a82","modified":1689472919651},{"_id":"public/archives/page/3/index.html","hash":"e8b98989cad57551d8ad7b0d8eee6c84b30fefd2","modified":1689472919651},{"_id":"public/archives/page/5/index.html","hash":"6d40d7d07b8c0065eac32c9121a2ac76807a467a","modified":1689472919651},{"_id":"public/archives/page/4/index.html","hash":"5ab7db2e0b5be461fa818a097e8fe009a3c82d5d","modified":1689472919651},{"_id":"public/archives/2022/page/2/index.html","hash":"54385204b0d6b4be7cf28b147f3204dba926d2fc","modified":1689472919651},{"_id":"public/archives/2022/index.html","hash":"b467099362008051a25579e6e3aeb98b73f6607c","modified":1689472919651},{"_id":"public/archives/2022/09/index.html","hash":"f176789d8a9c4ed0b0e9cbc8f404d571f791cfca","modified":1689472919651},{"_id":"public/archives/2022/11/index.html","hash":"d49618a16927a99368faea401d30543f5723c1ad","modified":1689472919651},{"_id":"public/archives/2022/12/index.html","hash":"981b46655bcb5d6dfb0890921e6aae596cdd9038","modified":1689472919651},{"_id":"public/archives/2023/index.html","hash":"1f67c3ba44d84b7dc88ee264ab4c5c4fca5be704","modified":1689472919651},{"_id":"public/archives/2022/10/index.html","hash":"e5cea227099daeab8dcacb78f61cbabbbc5c0c5a","modified":1689472919651},{"_id":"public/archives/2023/page/2/index.html","hash":"ae51bdbd0e3cb50ca1744919e95e98a682aaa70b","modified":1689472919651},{"_id":"public/archives/2023/01/index.html","hash":"977f9a56c4bf5fe390620f24c41fce09c4173d47","modified":1689472919651},{"_id":"public/archives/2023/03/index.html","hash":"9440a27627315849566fd06e849481011e120dc9","modified":1689472919651},{"_id":"public/archives/2023/page/3/index.html","hash":"26211e1e5fe93ff2950e6d2d737416e05ae29147","modified":1689472919651},{"_id":"public/archives/2023/04/index.html","hash":"2a077387598b32504cf4d4c7c426c113217016e2","modified":1689472919651},{"_id":"public/archives/2023/05/index.html","hash":"c32aff8fab114137fbec30ae81e583dce3142bcd","modified":1689472919651},{"_id":"public/archives/2023/06/index.html","hash":"e016132f45792b16e758b363bd8ee94779caf76d","modified":1689472919651},{"_id":"public/archives/2023/02/index.html","hash":"8a394f9ed98d47b88090f8c45ed11986a85d8521","modified":1689472919651},{"_id":"public/page/5/index.html","hash":"1f6fcf361b2beb680c1c2ad10e247bdef4389c8a","modified":1689472919651},{"_id":"public/tags/blockchain/index.html","hash":"89abcff12636efb65f3da190a57bb3d0395790ab","modified":1689472919651},{"_id":"public/tags/cryptography/index.html","hash":"6532dec4328bbb6a52aff844704b719b47b100aa","modified":1689472919651},{"_id":"public/tags/golang/index.html","hash":"342cc5e3aa22661867408ac16f3da47e5db5c761","modified":1689472919651},{"_id":"public/tags/mpc/index.html","hash":"7ec2ed2113454013670b9d6c144ccb3dd0ee78f1","modified":1689472919651},{"_id":"public/tags/ecdsa/index.html","hash":"2f5c8bbc9597090f3ebece706ad4da62b2bf73d8","modified":1689472919651},{"_id":"public/tags/rust/page/2/index.html","hash":"15719a6f1fc78083d3313c1ef045fd97cdae2157","modified":1689472919651},{"_id":"public/2023/06/20/cryptography/cryptography-04-digital-signature/index.html","hash":"c3973b5a56c04b98aa90404368a798566f788e25","modified":1689472919651},{"_id":"public/2023/06/10/cryptography/cryptography-02-rsa/index.html","hash":"47ac173aa160d7c61e0beffdfc44c3094ac827ba","modified":1689472919651},{"_id":"public/2023/06/08/rust/rust_std/rust-std-sync/index.html","hash":"df03bfc97db2780f6f880dde057ab2bb200371de","modified":1689472919651},{"_id":"public/2023/06/03/rust/rust_std/rust-smart-pointer-and-internal-mutibility/index.html","hash":"83af2a0883cc665c6ee9e32ddde0e69aad6a9f58","modified":1689472919651},{"_id":"public/2023/06/03/cryptography/cryptography-01-primitive-group-and-field/index.html","hash":"2c874cebec027b141dcf012bad7f444e19dee5b0","modified":1689472919651},{"_id":"public/2023/05/02/rust/rust_std/rust-std-data-structure-2/index.html","hash":"7db4fa0cc19834d915febd32df303744a3de0b78","modified":1689472919651},{"_id":"public/2023/05/01/rust/rust_std/rust-std-data-structure-1/index.html","hash":"9df626af5f700c9e75e787197d1ef33554f30c0a","modified":1689472919651},{"_id":"public/2023/03/18/geth/tech_docs/geth.sync.mode/index.html","hash":"9c0c38272d2e9dd2b64828f7516b1f7e007737e7","modified":1689472919651},{"_id":"public/2023/03/15/geth/tech_docs/geth.v1.10.0/index.html","hash":"08b29d40e3b9ef9ce75ae531bcd141d782c2bb3b","modified":1689472919651},{"_id":"public/2023/02/23/cryptography/paillier-encryption/index.html","hash":"61ceaa4c8ef6edd50133b5c49ab1f069396a0408","modified":1689472919651},{"_id":"public/2023/03/02/golang/go-reflect/index.html","hash":"c788f8f32d3ea5b382413244f7146f63fb2df632","modified":1689472919651},{"_id":"public/2023/01/15/blockchain.kademlia/index.html","hash":"a8101c8672dd67fe8eec5f272bcf9b86211e9d37","modified":1689472919651},{"_id":"public/2023/01/13/rust/rust-async/index.html","hash":"892dd88e065f9b0459635e4229622b7204d31278","modified":1689472919651},{"_id":"public/2023/01/08/geth/code_analysis/geth.evm/index.html","hash":"dbd13ba2a9bf672bb40549512ea1fea6cc9d0564","modified":1689472919651},{"_id":"public/2022/12/28/rust/rust-07-macro/index.html","hash":"2761e391c36b639f933f5516660619b7876b9021","modified":1689472919651},{"_id":"public/2022/12/06/rust/rust-cargo-all-in-one/index.html","hash":"369e1f8694d46f2500d55ff1d7d961c7b3977aa8","modified":1689472919651},{"_id":"public/2022/11/23/rust/rust-similar-concepts-comparison/index.html","hash":"459e41e67c693654b8345c01da8dfc2e0b702b2b","modified":1689472919651},{"_id":"public/2022/11/17/rust/rust-sugar/index.html","hash":"c24b0011264b27b00639bbc6d3c07707bc5888ed","modified":1689472919651},{"_id":"public/2022/11/06/rust/rust-08-project-management/index.html","hash":"b07aed3af4e371435a8f90a3005f53f1324c3064","modified":1689472919651},{"_id":"public/2022/11/01/geth/code_analysis/geth.0.get.start/index.html","hash":"34f05d47f87e0265d7138686993f6b54b66121f7","modified":1689472919651},{"_id":"public/2022/10/30/rust/rust-06-smart-pointer/index.html","hash":"7e76436e90bca326f126442cbc1775bed0f020c8","modified":1689472919651},{"_id":"public/2022/10/18/rust/rust-04-lifetime/index.html","hash":"fac539b49c57a1fdc3587c037d98d1a5ee8aff9a","modified":1689472919651},{"_id":"public/2022/10/11/rust/rust-03-ownership/index.html","hash":"fc0f99b144e1dd33ece3edfcd98e309d8a3d4bc3","modified":1689472919651},{"_id":"public/2022/10/04/rust/rust-02-basics/index.html","hash":"5ec5e7ea1deb38fb2f89cb7221393ef7b156a5b7","modified":1689472919651},{"_id":"public/page/2/index.html","hash":"05aedb9ca260c4995bb27b88a293d2035dc83317","modified":1689472919651},{"_id":"public/page/3/index.html","hash":"37538a9fe913d20e0f5060967da2a5d212e698cd","modified":1689472919651},{"_id":"public/page/4/index.html","hash":"b3ea512c468069bbf059a6eb740911b7a0893785","modified":1689472919651},{"_id":"public/index.html","hash":"7b2fd829571a639ef00312b5dc641c042bfcd01f","modified":1689472919651},{"_id":"public/tags/rust/index.html","hash":"07547f596d766280c9681b4ae44b49cab7091e12","modified":1689472919651},{"_id":"public/tags/cargo/index.html","hash":"860636559ec2e3b3e2cc029cd47b9209191776b3","modified":1689472919651},{"_id":"public/tags/zkp/index.html","hash":"c819b61155ce6bbba2734578d2fbfb28e122b190","modified":1689472919651},{"_id":"public/tags/rust-crate/index.html","hash":"49e5935cab7b92bda4278968a87b74c821063872","modified":1689472919651},{"_id":"public/tags/geth/index.html","hash":"631bf4a59672550dc83ae2c950fe39d8d20483f6","modified":1689472919651},{"_id":"public/tags/rust-std/index.html","hash":"d9138f26b2219e1b1c30ceeda75903e747cee86d","modified":1689472919651},{"_id":"public/images/evm.layout.png","hash":"6927cb3b922b2bcec926ec5d797e7ea8ea2b5d00","modified":1689472919651},{"_id":"public/images/kademlia.onlineprob.png","hash":"98c55aa819ef7047b3749c89eb4546f0d799a239","modified":1689472919651},{"_id":"public/images/mpt.state.ref.png","hash":"ec747cf092820c0ab5f72d0f4f07f7705bb9ecf0","modified":1689472919651},{"_id":"public/images/geth/sync.mode.jpg","hash":"657cc148abc14f01e7483265c67936e73f79d0e4","modified":1689472919651},{"_id":"public/images/paillier/homomorphic_addition.png","hash":"26b0e4a070a7e29710e2ceace09bbdb79b3a883e","modified":1689472919651},{"_id":"public/images/paillier/homomorphic_mul.png","hash":"e8a2420501140a8d48db202065aa221df92f19dc","modified":1689472919651},{"_id":"public/images/two_party_ecdsa/paillier_enc.png","hash":"132cdc74eb588951d1bd347b1b6e825912bc0460","modified":1689472919651},{"_id":"public/images/cryptography/elliptic_curve/point_addition.webp","hash":"ba6cd29aec4a694b53933d0824df180158f98316","modified":1689472919651},{"_id":"public/images/rust/ownership/move-string.png","hash":"d708edd6fb84b9d4ebe75556932f6b920934ca9a","modified":1689472919651},{"_id":"public/images/rust/memory/trait_object_memory.png","hash":"dd750cffa7bca5bde7856cfb65f4500de286c96f","modified":1689472919651},{"_id":"public/images/rust/pointers/rc.png","hash":"f568f69808ef0357ea5e6d42667d863b1139cbf1","modified":1689472919651},{"_id":"public/js/script.js","hash":"998ed4c5b147e1299bf62beebf33514474f28112","modified":1689472919651},{"_id":"public/fancybox/jquery.fancybox.min.css","hash":"1be9b79be02a1cfc5d96c4a5e0feb8f472babd95","modified":1689472919651},{"_id":"public/css/style.css","hash":"4da345d832a2682bcaee3ab3e22c15e3cd0e9cde","modified":1689472919651},{"_id":"public/js/jquery-3.4.1.min.js","hash":"88523924351bac0b5d560fe0c5781e2556e7693d","modified":1689472919651},{"_id":"public/fancybox/jquery.fancybox.min.js","hash":"6181412e73966696d08e1e5b1243a572d0f22ba6","modified":1689472919651},{"_id":"public/css/fonts/fontawesome-webfont.woff2","hash":"d6f48cba7d076fb6f2fd6ba993a75b9dc1ecbf0c","modified":1689472919651},{"_id":"public/css/fonts/fontawesome-webfont.woff","hash":"28b782240b3e76db824e12c02754a9731a167527","modified":1689472919651},{"_id":"public/images/evm.drawio.google.png","hash":"413a44d66153c93e2c31172d988e051a2bed9940","modified":1689472919651},{"_id":"public/images/geth_starts.drawio.png","hash":"3c426bd608121669f232aaa1b05ed7a342287fc8","modified":1689472919651},{"_id":"public/images/trie.prefix.png","hash":"67d26d416faf0cc43ee35b4acda598e88dad2949","modified":1689472919651},{"_id":"public/images/kademlia.subtree.png","hash":"d3198467460972151f4b2fe2b56fab0dd9e411ca","modified":1689472919651},{"_id":"public/images/paillier/carmichael_thorem.png","hash":"75b4d32eb2233db653bca52cdea4a624555b5ce4","modified":1689472919651},{"_id":"public/images/paillier/carmichael_thorem_2.png","hash":"29b103c94c36f4520ca8b675486af3d914998ac1","modified":1689472919651},{"_id":"public/images/rust/ownership/move-string-2.png","hash":"83342aed7ee254c099ada5d49ed8b9ba52a451a0","modified":1689472919651},{"_id":"public/images/two_party_ecdsa/schnorr_ecdsa_comparison.png","hash":"69cbed302af467a4e99653dfb51dca45c4a5a6f3","modified":1689472919651},{"_id":"public/images/rust/pointers/cycle.ref.png","hash":"ed99e2c6020833ccd5af751cf6eb2031cf47d9aa","modified":1689472919651},{"_id":"public/css/fonts/fontawesome-webfont.eot","hash":"d980c2ce873dc43af460d4d572d441304499f400","modified":1689472919651},{"_id":"public/css/fonts/FontAwesome.otf","hash":"048707bc52ac4b6563aaa383bfe8660a0ddc908c","modified":1689472919651},{"_id":"public/css/fonts/fontawesome-webfont.ttf","hash":"13b1eab65a983c7a73bc7997c479d66943f7c6cb","modified":1689472919651},{"_id":"public/css/images/banner.jpg","hash":"f44aa591089fcb3ec79770a1e102fd3289a7c6a6","modified":1689472919651},{"_id":"public/images/kademlia.locating.png","hash":"702d6b779294a3c6e033cc9bde14ef8950982310","modified":1689472919651},{"_id":"public/images/rust/macros/16.compile_process.png","hash":"444080319ca2101672941346988f78ed9a37c32d","modified":1689472919651},{"_id":"public/images/cryptography/rsa/rsa_signature.png","hash":"44eb1a608dafaae244e487cf082aa79c2af5c19f","modified":1689472919651},{"_id":"public/images/mpt.png","hash":"700032035bdcd793f94da522330552727b00e5a3","modified":1689472919651},{"_id":"public/css/fonts/fontawesome-webfont.svg","hash":"98a8aa5cf7d62c2eff5f07ede8d844b874ef06ed","modified":1689472919651},{"_id":"public/2017-552.pdf","hash":"b1a08857c1f6532f1fbb718c48a34fd48ea7da70","modified":1689472919651}],"Category":[],"Data":[],"Page":[],"Post":[{"title":"MPT","date":"2023-01-22T09:25:36.000Z","_content":"\n# trie\na trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.\n\n![prefix trie](/images/trie.prefix.png)\nIn general, the nodes of a Trie look like this:\n```\n[ [Ia, Ib, … I*], value]\n```\n[Ia, Ib, ... I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value\n\n# MPT\nEach block of Ethereum contains three MPT trees, respectively\n\n- Transaction tree\n- Receipt tree\n- State tree\n\nIn the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is\n\n![state reference](/images/mpt.state.ref.png)\n\n- use []byte as key, other than string\n- nibble: the smallest unit of the key type (4 bit)\n- Use hashes to refer to nodes instead of memory pointers\n\nthere are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows\n```\nfullNode: [i0, i1, i2, … i15, hash] \nshortNode： [ [k0, k1, … kn], hash ] // first element is an array\n```\nif the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.\n\n![mpt](/images/mpt.png)\n\nUse the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value\n|hex char| bits | pointing to | odd/even | 2nd niddle padding |\n| -----|:----:|:----:|:----:|-------|\n|0| 0000 | node | even | no |\n|1| 0001 | node | odd | yes |\n|2| 0010 | value | even | no |\n|3| 0011 | value | odd | yes |\n\nthis encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type\n\nIn the trie module, there is a `Database` object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database\n\n# reference\n- [github](https://github.com/agiletechvn/go-ethereum-code-analysis/blob/master/trie-analysis.md)\n- [yangzhe's blog](http://yangzhe.me/2019/01/12/ethereum-trie-part-1/)\n- [yangzhe's blod](http://yangzhe.me/2019/01/18/ethereum-trie-part-2/)","source":"_posts/MPT.md","raw":"---\ntitle: MPT\ndate: 2023-01-22 17:25:36\ntags: [blockchain, geth]\n---\n\n# trie\na trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.\n\n![prefix trie](/images/trie.prefix.png)\nIn general, the nodes of a Trie look like this:\n```\n[ [Ia, Ib, … I*], value]\n```\n[Ia, Ib, ... I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value\n\n# MPT\nEach block of Ethereum contains three MPT trees, respectively\n\n- Transaction tree\n- Receipt tree\n- State tree\n\nIn the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is\n\n![state reference](/images/mpt.state.ref.png)\n\n- use []byte as key, other than string\n- nibble: the smallest unit of the key type (4 bit)\n- Use hashes to refer to nodes instead of memory pointers\n\nthere are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows\n```\nfullNode: [i0, i1, i2, … i15, hash] \nshortNode： [ [k0, k1, … kn], hash ] // first element is an array\n```\nif the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.\n\n![mpt](/images/mpt.png)\n\nUse the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value\n|hex char| bits | pointing to | odd/even | 2nd niddle padding |\n| -----|:----:|:----:|:----:|-------|\n|0| 0000 | node | even | no |\n|1| 0001 | node | odd | yes |\n|2| 0010 | value | even | no |\n|3| 0011 | value | odd | yes |\n\nthis encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type\n\nIn the trie module, there is a `Database` object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database\n\n# reference\n- [github](https://github.com/agiletechvn/go-ethereum-code-analysis/blob/master/trie-analysis.md)\n- [yangzhe's blog](http://yangzhe.me/2019/01/12/ethereum-trie-part-1/)\n- [yangzhe's blod](http://yangzhe.me/2019/01/18/ethereum-trie-part-2/)","slug":"MPT","published":1,"updated":"2023-04-15T04:52:00.319Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzze0000ofsjat5l27fl","content":"

trie

a trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.

\n

\"prefix
In general, the nodes of a Trie look like this:

\n
1
[ [Ia, Ib, … I*], value]
\n

[Ia, Ib, … I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value

\n

MPT

Each block of Ethereum contains three MPT trees, respectively

\n
    \n
  • Transaction tree
    • \n
  • Receipt tree
    • \n
  • State tree
    • \n
\n

In the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is

\n

\"state

\n
    \n
  • use []byte as key, other than string
    • \n
  • nibble: the smallest unit of the key type (4 bit)
    • \n
  • Use hashes to refer to nodes instead of memory pointers
    • \n
\n

there are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows

\n
1
2
fullNode: [i0, i1, i2, … i15, hash]  
shortNode： [ [k0, k1, … kn], hash ] // first element is an array
\n

if the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.

\n

\"mpt\"

\n

Use the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
hex charbitspointing toodd/even2nd niddle padding
00000nodeevenno
10001nodeoddyes
20010valueevenno
30011valueoddyes
\n

this encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type

\n

In the trie module, there is a Database object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database

\n

reference

\n","site":{"data":{}},"excerpt":"","more":"

trie

a trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.

\n

\"prefix
In general, the nodes of a Trie look like this:

\n
1
[ [Ia, Ib, … I*], value]
\n

[Ia, Ib, … I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value

\n

MPT

Each block of Ethereum contains three MPT trees, respectively

\n
    \n
  • Transaction tree
    • \n
  • Receipt tree
    • \n
  • State tree
    • \n
\n

In the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is

\n

\"state

\n
    \n
  • use []byte as key, other than string
    • \n
  • nibble: the smallest unit of the key type (4 bit)
    • \n
  • Use hashes to refer to nodes instead of memory pointers
    • \n
\n

there are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows

\n
1
2
fullNode: [i0, i1, i2, … i15, hash]  
shortNode： [ [k0, k1, … kn], hash ] // first element is an array
\n

if the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.

\n

\"mpt\"

\n

Use the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
hex charbitspointing toodd/even2nd niddle padding
00000nodeevenno
10001nodeoddyes
20010valueevenno
30011valueoddyes
\n

this encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type

\n

In the trie module, there is a Database object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database

\n

reference

\n"},{"title":"understanding of kademlia protocol","date":"2023-01-15T03:00:30.000Z","_content":"\n# Introduction\nKademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) pairs are stored in nodes whose ID is 'close' to the key for some notion of closeness. \n\n# system description\nKad effectively treats nodes as leaves in a binary tree, with each nodes's position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.\n\n![Figure 1](/images/kademlia.subtree.png)\nfor any given node, the binary tree is divided into a series of successively lower subtrees that don't contain the node.\n The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.\n\nif there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.\n\n![Figure 2](/images/kademlia.locating.png)\n\n## node distance: XOR metric\nNode's id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.\nd(x,y) = x XOR y\n\n## node state\nfor each 0 <= i < 160, every node keeps k triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.\neach k-bucket is kept sorted by time last seen--least recently seen node at the head, most-recently seen at the tail.\nfor small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).\n\n The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8). \n\n## update of k bucket\nThere are mainly the following ways to update the K bucket:\n\n- Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.\n- Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.\n- Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.\n\nThe update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.\n\nAccording to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes\n![probability of continuous online agains onlie duration](/images/kademlia.onlineprob.png)\n\n## RPC method\nThe Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.\n\n- PING probes a node to see if it is online.\n- STORE instructs a node to store a pair for later retrieval.\n- FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator. \n- The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.\n\n## node lookup\nKad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of. \n\n## find a pair\nto find a pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.\n\n## node join\nto join the network, a node u must have a contact to an already participating node w.\n\n## routing table\n\n# references\n- [kademlia paper](https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf)\n- [go implementation](https://github.com/libp2p/go-libp2p-kad-dht)\n- [kademlia stanford](https://codethechange.stanford.edu/guides/guide_kademlia.html)\n- [zhihu](https://zhuanlan.zhihu.com/p/388994038)","source":"_posts/blockchain.kademlia.md","raw":"---\ntitle: understanding of kademlia protocol\ndate: 2023-01-15 11:00:30\ntags: [blockchain]\n---\n\n# Introduction\nKademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) pairs are stored in nodes whose ID is 'close' to the key for some notion of closeness. \n\n# system description\nKad effectively treats nodes as leaves in a binary tree, with each nodes's position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.\n\n![Figure 1](/images/kademlia.subtree.png)\nfor any given node, the binary tree is divided into a series of successively lower subtrees that don't contain the node.\n The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.\n\nif there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.\n\n![Figure 2](/images/kademlia.locating.png)\n\n## node distance: XOR metric\nNode's id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.\nd(x,y) = x XOR y\n\n## node state\nfor each 0 <= i < 160, every node keeps k triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.\neach k-bucket is kept sorted by time last seen--least recently seen node at the head, most-recently seen at the tail.\nfor small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).\n\n The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8). \n\n## update of k bucket\nThere are mainly the following ways to update the K bucket:\n\n- Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.\n- Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.\n- Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.\n\nThe update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.\n\nAccording to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes\n![probability of continuous online agains onlie duration](/images/kademlia.onlineprob.png)\n\n## RPC method\nThe Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.\n\n- PING probes a node to see if it is online.\n- STORE instructs a node to store a pair for later retrieval.\n- FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator. \n- The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.\n\n## node lookup\nKad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of. \n\n## find a pair\nto find a pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.\n\n## node join\nto join the network, a node u must have a contact to an already participating node w.\n\n## routing table\n\n# references\n- [kademlia paper](https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf)\n- [go implementation](https://github.com/libp2p/go-libp2p-kad-dht)\n- [kademlia stanford](https://codethechange.stanford.edu/guides/guide_kademlia.html)\n- [zhihu](https://zhuanlan.zhihu.com/p/388994038)","slug":"blockchain.kademlia","published":1,"updated":"2023-04-14T07:33:16.860Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzh0001ofsj9gqng05l","content":"

Introduction

Kademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) <key,value> pairs are stored in nodes whose ID is ‘close’ to the key for some notion of closeness.

\n

system description

Kad effectively treats nodes as leaves in a binary tree, with each nodes’s position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.

\n

\"Figure
for any given node, the binary tree is divided into a series of successively lower subtrees that don’t contain the node.
The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.

\n

if there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.

\n

\"Figure

\n

node distance: XOR metric

Node’s id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.
d(x,y) = x XOR y

\n

node state

for each 0 <= i < 160, every node keeps k <IP address, UDP port, node ID> triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.
each k-bucket is kept sorted by time last seen–least recently seen node at the head, most-recently seen at the tail.
for small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).

\n

The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8).

\n

update of k bucket

There are mainly the following ways to update the K bucket:

\n
    \n
  • Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.
    • \n
  • Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.
    • \n
  • Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.
    • \n
\n

The update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.

\n

According to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes
\"probability

\n

RPC method

The Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.

\n
    \n
  • PING probes a node to see if it is online.
    • \n
  • STORE instructs a node to store a <key,value> pair for later retrieval.
    • \n
  • FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator.
    • \n
  • The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.
    • \n
\n

node lookup

Kad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of.

\n

find a <key,value> pair

to find a <key,value> pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.

\n

node join

to join the network, a node u must have a contact to an already participating node w.

\n

routing table

references

\n","site":{"data":{}},"excerpt":"","more":"

Introduction

Kademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) <key,value> pairs are stored in nodes whose ID is ‘close’ to the key for some notion of closeness.

\n

system description

Kad effectively treats nodes as leaves in a binary tree, with each nodes’s position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.

\n

\"Figure
for any given node, the binary tree is divided into a series of successively lower subtrees that don’t contain the node.
The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.

\n

if there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.

\n

\"Figure

\n

node distance: XOR metric

Node’s id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.
d(x,y) = x XOR y

\n

node state

for each 0 <= i < 160, every node keeps k <IP address, UDP port, node ID> triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.
each k-bucket is kept sorted by time last seen–least recently seen node at the head, most-recently seen at the tail.
for small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).

\n

The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8).

\n

update of k bucket

There are mainly the following ways to update the K bucket:

\n
    \n
  • Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.
    • \n
  • Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.
    • \n
  • Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.
    • \n
\n

The update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.

\n

According to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes
\"probability

\n

RPC method

The Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.

\n
    \n
  • PING probes a node to see if it is online.
    • \n
  • STORE instructs a node to store a <key,value> pair for later retrieval.
    • \n
  • FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator.
    • \n
  • The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.
    • \n
\n

node lookup

Kad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of.

\n

find a <key,value> pair

to find a <key,value> pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.

\n

node join

to join the network, a node u must have a contact to an already participating node w.

\n

routing table

references

\n"},{"title":"geth_fine_tune","date":"2023-01-01T08:29:43.000Z","_content":"\n\nIf we're a full node on mainnet without --cache specified, bump default cache allowance\nctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))","source":"_posts/geth-fine-tune.md","raw":"---\ntitle: geth_fine_tune\ndate: 2023-01-01 16:29:43\ntags:\n---\n\n\nIf we're a full node on mainnet without --cache specified, bump default cache allowance\nctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))","slug":"geth-fine-tune","published":1,"updated":"2023-04-25T09:48:14.119Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzj0003ofsj6iqx9blc","content":"

If we’re a full node on mainnet without –cache specified, bump default cache allowance
ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))

\n","site":{"data":{}},"excerpt":"","more":"

If we’re a full node on mainnet without –cache specified, bump default cache allowance
ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))

\n"},{"title":"how to use hexo","date":"2022-11-27T03:47:56.000Z","_content":"Welcome to [Hexo](https://hexo.io/)! This is your very first post. Check [documentation](https://hexo.io/docs/) for more info. If you get any problems when using Hexo, you can find the answer in [troubleshooting](https://hexo.io/docs/troubleshooting.html) or you can ask me on [GitHub](https://github.com/hexojs/hexo/issues).\n\n## Quick Start\n\n### Create a new post\n\n``` bash\n$ hexo new \"My New Post\"\n```\n\nMore info: [Writing](https://hexo.io/docs/writing.html)\n\n### Run server\n\n``` bash\n$ hexo server\n```\n\nMore info: [Server](https://hexo.io/docs/server.html)\n\n### Generate static files\n\n``` bash\n$ hexo generate\n```\n\nMore info: [Generating](https://hexo.io/docs/generating.html)\n\n### Deploy to remote sites\n\n``` bash\n$ hexo deploy\n```\n\nMore info: [Deployment](https://hexo.io/docs/one-command-deployment.html)\n","source":"_posts/hello-world.md","raw":"---\ntitle: how to use hexo\ndate: 2022-11-27 11:47:56\n---\nWelcome to [Hexo](https://hexo.io/)! This is your very first post. Check [documentation](https://hexo.io/docs/) for more info. If you get any problems when using Hexo, you can find the answer in [troubleshooting](https://hexo.io/docs/troubleshooting.html) or you can ask me on [GitHub](https://github.com/hexojs/hexo/issues).\n\n## Quick Start\n\n### Create a new post\n\n``` bash\n$ hexo new \"My New Post\"\n```\n\nMore info: [Writing](https://hexo.io/docs/writing.html)\n\n### Run server\n\n``` bash\n$ hexo server\n```\n\nMore info: [Server](https://hexo.io/docs/server.html)\n\n### Generate static files\n\n``` bash\n$ hexo generate\n```\n\nMore info: [Generating](https://hexo.io/docs/generating.html)\n\n### Deploy to remote sites\n\n``` bash\n$ hexo deploy\n```\n\nMore info: [Deployment](https://hexo.io/docs/one-command-deployment.html)\n","slug":"hello-world","published":1,"updated":"2023-04-06T16:43:39.107Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzk0004ofsj4mko7y06","content":"

Welcome to Hexo! This is your very first post. Check documentation for more info. If you get any problems when using Hexo, you can find the answer in troubleshooting or you can ask me on GitHub.

\n

Quick Start

Create a new post

1
$ hexo new "My New Post"
\n\n

More info: Writing

\n

Run server

1
$ hexo server
\n\n

More info: Server

\n

Generate static files

1
$ hexo generate
\n\n

More info: Generating

\n

Deploy to remote sites

1
$ hexo deploy
\n\n

More info: Deployment

\n","site":{"data":{}},"excerpt":"","more":"

Welcome to Hexo! This is your very first post. Check documentation for more info. If you get any problems when using Hexo, you can find the answer in troubleshooting or you can ask me on GitHub.

\n

Quick Start

Create a new post

1
$ hexo new "My New Post"
\n\n

More info: Writing

\n

Run server

1
$ hexo server
\n\n

More info: Server

\n

Generate static files

1
$ hexo generate
\n\n

More info: Generating

\n

Deploy to remote sites

1
$ hexo deploy
\n\n

More info: Deployment

\n"},{"title":"cryptography (1) group & field","date":"2023-06-03T06:29:26.000Z","_content":"\n\n\n\n\n## Group\na group is a set of elements \\\\( G \\\\) together with an operation \\\\( \\circ \\\\). a group has the following properties\n1. the group operation \\\\( \\circ \\\\) is closed. that is, for all \\\\( a,b, \\in G \\\\), it holds that \\\\( a \\circ b = c \\in G \\\\)\n2. the group operation is associative. That is, \\\\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\\\) for all \\\\( a,b,c \\in G \\\\)\n3. there is an element \\\\( 1 \\in G \\\\), called the neutral element (or identity element), such that \\\\( a \\circ \\ = 1 \\circ a\\\\) for all \\\\( a \\in G \\\\)\n4. For each \\\\( a \\in G \\\\) there exists an element \\\\( a^{-1} \\in G\\\\), called the inverse of a, such that \\\\( a \\circ a^{-1} = 1 \\\\).\n5. a group G is abelian (or commutative) if, furthermore, \\\\( a \\circ b = b \\circ a \\\\) for all \\\\( a, b \\in G \\\\) \n\n### Example of Group\nthe set of integers \\\\( Z_{m} = 0,1,...,m-1 \\\\) and the operation addition modulo \\\\( m \\\\) forma group with the neutral element 0. every element \\\\( a \\\\) has an inverse \\\\( -a \\\\). note that this set does not form a group with the operation multiplication gecause mose eleents \\\\( a \\\\) do not have an inverse such that \\\\( a a^{-1} = 1 \\bmod m\\\\).\n\nIn order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.\n\n## Field\nA field is a set of elements with the following properties\n- all elements of \\\\( F\\\\) form an additive group with the group operation \\\\( + \\\\) and the neutral element \\\\( 0 \\\\)\n- all element of \\\\( F \\\\) except \\\\( 0 \\\\) form a multiplicative group with the gorup operation \\\\( x \\\\) and the neutral element \\\\( 1 \\\\) .\n- when the two group operations are mixed, the distributivety law holds, i.e., for all \\\\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\\\) \n\n### Example of Field\nthe set \\\\( R \\\\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\\\( a \\\\) has an additive inverse, namely \\\\( -a \\\\), and every nonzero element \\\\( a \\\\) has a multiplicative inverse \\\\( 1 \\div a \\\\)\n\n## Galois Field\nIn cryptography, we are almose always interested in fields with a **finite** number of elements, which we call finite fields or `Galois field`. the number of elements in the field is called the `order` or `cardinality` of the field. Of fundamental importance is the following theorem\n***\na field with order `m` only exists if `m` is a prime power, i.e, \\\\( m = p^{n} \\\\), for some positive integer `n` and prime integer `p`. `p` is called the characteristic of the finite field\n***\n\nthe theorem implies that there are, for instance, finite fields with 11 elements (\\\\( 11^{1} \\\\)), or with 81 elements \\\\( 81 = 3^{4} \\\\) or with 256 elements (\\\\( 256 = 2^{8} \\\\)). \n\n## Prime Field\nthe most intuitive examples of finite fields are fields of prime order, i.e., fields with \\\\( n =1 \\\\). elements of the field \\\\( GF(p) \\\\) can be represented by integers \\\\( 0,1,..., p-1 \\\\).\n\n## Extension Field `GF(2^m)`\nIn AES the finite field contains 256 elements and is denoted as \\\\( GF(2^8) \\\\).\nHowever, if the order of a finite field is not prime, and \\\\( 2^8 \\\\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\\\( 2^8 \\\\). such fields with \\\\( m >1 \\\\) are called `extension field`. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as `polynomials`, and computation in the extension field is achieved by performing a certain type of `polynomial arithmetic`.\n\nIn the field \\\\( GF(2^8) \\\\), which is used in AES, each element \\\\( A \\in GF(2^8) \\\\) is thus represented as: \n\\\\[ A(x) = a_{7}x^7 + ...+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\\\]\nIt is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector\n\\\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\\\]\n\n### addition and subtraction in `GF(2^m)`\naddition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.\n\\\\[ A(x) = x^7 + x^6 + x^4 + 1 \\\\]\n\\\\[ B(x) = x^4 + x^2+ 1 \\\\]\n\\\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\\\]\n\n### multiplication in `GF(2^m)`\nfirstly, two elements (represented by their polynomials) of a finite field `GF(2^m)` are multiplied usig the standard polynomial multiplication rule \\\\[ A(x) \\dot B(x) = C(x) \\\\]. In general, the product polynomial \\\\[ C(x) \\\\] will have a degree higher than `m-1` and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need **irreducible** polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.\nThus, every field `GF(2^m)` requires an irreducible polynomial `P(x)` of degree `m`. \nFor AES, the irreducible polynomial is \n\\\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\\\]\nthe main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.","source":"_posts/cryptography/cryptography-01-primitive-group-and-field.md","raw":"---\ntitle: cryptography (1) group & field\ndate: 2023-06-03 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n\n## Group\na group is a set of elements \\\\( G \\\\) together with an operation \\\\( \\circ \\\\). a group has the following properties\n1. the group operation \\\\( \\circ \\\\) is closed. that is, for all \\\\( a,b, \\in G \\\\), it holds that \\\\( a \\circ b = c \\in G \\\\)\n2. the group operation is associative. That is, \\\\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\\\) for all \\\\( a,b,c \\in G \\\\)\n3. there is an element \\\\( 1 \\in G \\\\), called the neutral element (or identity element), such that \\\\( a \\circ \\ = 1 \\circ a\\\\) for all \\\\( a \\in G \\\\)\n4. For each \\\\( a \\in G \\\\) there exists an element \\\\( a^{-1} \\in G\\\\), called the inverse of a, such that \\\\( a \\circ a^{-1} = 1 \\\\).\n5. a group G is abelian (or commutative) if, furthermore, \\\\( a \\circ b = b \\circ a \\\\) for all \\\\( a, b \\in G \\\\) \n\n### Example of Group\nthe set of integers \\\\( Z_{m} = 0,1,...,m-1 \\\\) and the operation addition modulo \\\\( m \\\\) forma group with the neutral element 0. every element \\\\( a \\\\) has an inverse \\\\( -a \\\\). note that this set does not form a group with the operation multiplication gecause mose eleents \\\\( a \\\\) do not have an inverse such that \\\\( a a^{-1} = 1 \\bmod m\\\\).\n\nIn order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.\n\n## Field\nA field is a set of elements with the following properties\n- all elements of \\\\( F\\\\) form an additive group with the group operation \\\\( + \\\\) and the neutral element \\\\( 0 \\\\)\n- all element of \\\\( F \\\\) except \\\\( 0 \\\\) form a multiplicative group with the gorup operation \\\\( x \\\\) and the neutral element \\\\( 1 \\\\) .\n- when the two group operations are mixed, the distributivety law holds, i.e., for all \\\\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\\\) \n\n### Example of Field\nthe set \\\\( R \\\\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\\\( a \\\\) has an additive inverse, namely \\\\( -a \\\\), and every nonzero element \\\\( a \\\\) has a multiplicative inverse \\\\( 1 \\div a \\\\)\n\n## Galois Field\nIn cryptography, we are almose always interested in fields with a **finite** number of elements, which we call finite fields or `Galois field`. the number of elements in the field is called the `order` or `cardinality` of the field. Of fundamental importance is the following theorem\n***\na field with order `m` only exists if `m` is a prime power, i.e, \\\\( m = p^{n} \\\\), for some positive integer `n` and prime integer `p`. `p` is called the characteristic of the finite field\n***\n\nthe theorem implies that there are, for instance, finite fields with 11 elements (\\\\( 11^{1} \\\\)), or with 81 elements \\\\( 81 = 3^{4} \\\\) or with 256 elements (\\\\( 256 = 2^{8} \\\\)). \n\n## Prime Field\nthe most intuitive examples of finite fields are fields of prime order, i.e., fields with \\\\( n =1 \\\\). elements of the field \\\\( GF(p) \\\\) can be represented by integers \\\\( 0,1,..., p-1 \\\\).\n\n## Extension Field `GF(2^m)`\nIn AES the finite field contains 256 elements and is denoted as \\\\( GF(2^8) \\\\).\nHowever, if the order of a finite field is not prime, and \\\\( 2^8 \\\\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\\\( 2^8 \\\\). such fields with \\\\( m >1 \\\\) are called `extension field`. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as `polynomials`, and computation in the extension field is achieved by performing a certain type of `polynomial arithmetic`.\n\nIn the field \\\\( GF(2^8) \\\\), which is used in AES, each element \\\\( A \\in GF(2^8) \\\\) is thus represented as: \n\\\\[ A(x) = a_{7}x^7 + ...+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\\\]\nIt is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector\n\\\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\\\]\n\n### addition and subtraction in `GF(2^m)`\naddition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.\n\\\\[ A(x) = x^7 + x^6 + x^4 + 1 \\\\]\n\\\\[ B(x) = x^4 + x^2+ 1 \\\\]\n\\\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\\\]\n\n### multiplication in `GF(2^m)`\nfirstly, two elements (represented by their polynomials) of a finite field `GF(2^m)` are multiplied usig the standard polynomial multiplication rule \\\\[ A(x) \\dot B(x) = C(x) \\\\]. In general, the product polynomial \\\\[ C(x) \\\\] will have a degree higher than `m-1` and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need **irreducible** polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.\nThus, every field `GF(2^m)` requires an irreducible polynomial `P(x)` of degree `m`. \nFor AES, the irreducible polynomial is \n\\\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\\\]\nthe main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.","slug":"cryptography/cryptography-01-primitive-group-and-field","published":1,"updated":"2023-07-13T16:01:11.422Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzk0005ofsj6z7xbxfl","content":"\n\n\n\n\n

Group

a group is a set of elements \\( G \\) together with an operation \\( \\circ \\). a group has the following properties

\n
    \n
  1. the group operation \\( \\circ \\) is closed. that is, for all \\( a,b, \\in G \\), it holds that \\( a \\circ b = c \\in G \\)
    2. \n
  2. the group operation is associative. That is, \\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\) for all \\( a,b,c \\in G \\)
    3. \n
  3. there is an element \\( 1 \\in G \\), called the neutral element (or identity element), such that \\( a \\circ \\ = 1 \\circ a\\) for all \\( a \\in G \\)
    4. \n
  4. For each \\( a \\in G \\) there exists an element \\( a^{-1} \\in G\\), called the inverse of a, such that \\( a \\circ a^{-1} = 1 \\).
    5. \n
  5. a group G is abelian (or commutative) if, furthermore, \\( a \\circ b = b \\circ a \\) for all \\( a, b \\in G \\)
    6. \n
\n

Example of Group

the set of integers \\( Z_{m} = 0,1,…,m-1 \\) and the operation addition modulo \\( m \\) forma group with the neutral element 0. every element \\( a \\) has an inverse \\( -a \\). note that this set does not form a group with the operation multiplication gecause mose eleents \\( a \\) do not have an inverse such that \\( a a^{-1} = 1 \\bmod m\\).

\n

In order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.

\n

Field

A field is a set of elements with the following properties

\n
    \n
  • all elements of \\( F\\) form an additive group with the group operation \\( + \\) and the neutral element \\( 0 \\)
    • \n
  • all element of \\( F \\) except \\( 0 \\) form a multiplicative group with the gorup operation \\( x \\) and the neutral element \\( 1 \\) .
    • \n
  • when the two group operations are mixed, the distributivety law holds, i.e., for all \\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\)
    • \n
\n

Example of Field

the set \\( R \\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\( a \\) has an additive inverse, namely \\( -a \\), and every nonzero element \\( a \\) has a multiplicative inverse \\( 1 \\div a \\)

\n

Galois Field

In cryptography, we are almose always interested in fields with a finite number of elements, which we call finite fields or Galois field. the number of elements in the field is called the order or cardinality of the field. Of fundamental importance is the following theorem

\n
\n

a field with order m only exists if m is a prime power, i.e, \\( m = p^{n} \\), for some positive integer n and prime integer p. p is called the characteristic of the finite field

\n
\n

the theorem implies that there are, for instance, finite fields with 11 elements (\\( 11^{1} \\)), or with 81 elements \\( 81 = 3^{4} \\) or with 256 elements (\\( 256 = 2^{8} \\)).

\n

Prime Field

the most intuitive examples of finite fields are fields of prime order, i.e., fields with \\( n =1 \\). elements of the field \\( GF(p) \\) can be represented by integers \\( 0,1,…, p-1 \\).

\n

Extension Field GF(2^m)

In AES the finite field contains 256 elements and is denoted as \\( GF(2^8) \\).
However, if the order of a finite field is not prime, and \\( 2^8 \\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\( 2^8 \\). such fields with \\( m >1 \\) are called extension field. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as polynomials, and computation in the extension field is achieved by performing a certain type of polynomial arithmetic.

\n

In the field \\( GF(2^8) \\), which is used in AES, each element \\( A \\in GF(2^8) \\) is thus represented as:
\\[ A(x) = a_{7}x^7 + …+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\]
It is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector
\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\]

\n

addition and subtraction in GF(2^m)

addition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.
\\[ A(x) = x^7 + x^6 + x^4 + 1 \\]
\\[ B(x) = x^4 + x^2+ 1 \\]
\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\]

\n

multiplication in GF(2^m)

firstly, two elements (represented by their polynomials) of a finite field GF(2^m) are multiplied usig the standard polynomial multiplication rule \\[ A(x) \\dot B(x) = C(x) \\]. In general, the product polynomial \\[ C(x) \\] will have a degree higher than m-1 and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need irreducible polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.
Thus, every field GF(2^m) requires an irreducible polynomial P(x) of degree m.
For AES, the irreducible polynomial is
\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\]
the main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n\n

Group

a group is a set of elements \\( G \\) together with an operation \\( \\circ \\). a group has the following properties

\n
    \n
  1. the group operation \\( \\circ \\) is closed. that is, for all \\( a,b, \\in G \\), it holds that \\( a \\circ b = c \\in G \\)
    2. \n
  2. the group operation is associative. That is, \\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\) for all \\( a,b,c \\in G \\)
    3. \n
  3. there is an element \\( 1 \\in G \\), called the neutral element (or identity element), such that \\( a \\circ \\ = 1 \\circ a\\) for all \\( a \\in G \\)
    4. \n
  4. For each \\( a \\in G \\) there exists an element \\( a^{-1} \\in G\\), called the inverse of a, such that \\( a \\circ a^{-1} = 1 \\).
    5. \n
  5. a group G is abelian (or commutative) if, furthermore, \\( a \\circ b = b \\circ a \\) for all \\( a, b \\in G \\)
    6. \n
\n

Example of Group

the set of integers \\( Z_{m} = 0,1,…,m-1 \\) and the operation addition modulo \\( m \\) forma group with the neutral element 0. every element \\( a \\) has an inverse \\( -a \\). note that this set does not form a group with the operation multiplication gecause mose eleents \\( a \\) do not have an inverse such that \\( a a^{-1} = 1 \\bmod m\\).

\n

In order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.

\n

Field

A field is a set of elements with the following properties

\n
    \n
  • all elements of \\( F\\) form an additive group with the group operation \\( + \\) and the neutral element \\( 0 \\)
    • \n
  • all element of \\( F \\) except \\( 0 \\) form a multiplicative group with the gorup operation \\( x \\) and the neutral element \\( 1 \\) .
    • \n
  • when the two group operations are mixed, the distributivety law holds, i.e., for all \\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\)
    • \n
\n

Example of Field

the set \\( R \\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\( a \\) has an additive inverse, namely \\( -a \\), and every nonzero element \\( a \\) has a multiplicative inverse \\( 1 \\div a \\)

\n

Galois Field

In cryptography, we are almose always interested in fields with a finite number of elements, which we call finite fields or Galois field. the number of elements in the field is called the order or cardinality of the field. Of fundamental importance is the following theorem

\n
\n

a field with order m only exists if m is a prime power, i.e, \\( m = p^{n} \\), for some positive integer n and prime integer p. p is called the characteristic of the finite field

\n
\n

the theorem implies that there are, for instance, finite fields with 11 elements (\\( 11^{1} \\)), or with 81 elements \\( 81 = 3^{4} \\) or with 256 elements (\\( 256 = 2^{8} \\)).

\n

Prime Field

the most intuitive examples of finite fields are fields of prime order, i.e., fields with \\( n =1 \\). elements of the field \\( GF(p) \\) can be represented by integers \\( 0,1,…, p-1 \\).

\n

Extension Field GF(2^m)

In AES the finite field contains 256 elements and is denoted as \\( GF(2^8) \\).
However, if the order of a finite field is not prime, and \\( 2^8 \\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\( 2^8 \\). such fields with \\( m >1 \\) are called extension field. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as polynomials, and computation in the extension field is achieved by performing a certain type of polynomial arithmetic.

\n

In the field \\( GF(2^8) \\), which is used in AES, each element \\( A \\in GF(2^8) \\) is thus represented as:
\\[ A(x) = a_{7}x^7 + …+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\]
It is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector
\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\]

\n

addition and subtraction in GF(2^m)

addition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.
\\[ A(x) = x^7 + x^6 + x^4 + 1 \\]
\\[ B(x) = x^4 + x^2+ 1 \\]
\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\]

\n

multiplication in GF(2^m)

firstly, two elements (represented by their polynomials) of a finite field GF(2^m) are multiplied usig the standard polynomial multiplication rule \\[ A(x) \\dot B(x) = C(x) \\]. In general, the product polynomial \\[ C(x) \\] will have a degree higher than m-1 and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need irreducible polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.
Thus, every field GF(2^m) requires an irreducible polynomial P(x) of degree m.
For AES, the irreducible polynomial is
\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\]
the main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.

\n"},{"title":"cryptography (2) RSA cryptosystem","date":"2023-06-10T06:29:26.000Z","_content":"\n\n\n\n## introduction\nthe security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].\n\n## Euclidean Algorithm\nthe gcd of two positive integers \\\\(r_0\\\\) and \\\\(r_1\\\\) is denoted by \n\\\\[gcd(r_0, r_1)\\\\]\nthe Eucliedean algorithm is used to calculate gcd, based on as simple observation that\n\\\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\\\]\nwhere we assume \\\\(r_0 > r_1\\\\)\nThis property can easily be proven: Let \\\\(gcd(r_0, r_1) = g\\\\), since \\\\(g\\\\) divides both \\\\(r_0\\\\) and \\\\(r_1\\\\), we can write \\\\(r_0 = g \\cdot x\\\\) and \\\\(r_1 = g \\cdot y\\\\), where \\\\(x>y\\\\) and \\\\(x\\\\) and \\\\(y\\\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\\\((x-y)\\\\) and \\\\(y\\\\) are also coprime. it follows from here that\n\\\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\\\]\n\nit also follow immediately that we can apply the process iteratively:\n\\\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\\\]\nas long as \\\\((r_0 - mr_1) >0\\\\). the algorithm use the fewest number of steps if we choose maximum value of \\\\(m\\\\). this is the case if we compute:\n\\\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\\\]\nthis process can be applied recursively until we obtain finally \\\\[gcd(r_l, 0) = r_l \\\\]\nthe euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.\n## Extended Euclidean Algorithm\nan extension of the Euclidean algorithm allows us to compute ***modular inverse***. in addition to computing the gcd, the ***extended Euclidean algorithm*** computes a linear combination of the form\n\\\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\\\]\nwhere s and t are integer coefficients. this equation is ofthen referred to as ***Diophantine equation***\n\nthe detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.\nlet \\\\(r_0 =973 , r_1 = 301\\\\). during the steps of Euclidean Algorithm, we obtain \\\\(973 = 3\\cdot301 + 70\\\\)\nwhich is \\\\[r_0 = q_1 \\cdot r_1 + r_2\\\\] \nrearrange:\n\\\\[r_2 = r_0 + (-q_1) \\cdot r_1\\\\] \nreplacing (r_0, r_1) and iteratively by (r_1, r_2), ... (r_{i-1}, r_{i}), util \\\\(r_{i+1} = 0\\\\)\nthen \\\\(r_{i}\\\\) is \\\\(gcd(r_0,r_1)\\\\), and can have a representation of \n\\\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\\\]. \nsince the inverse only exists if \\\\(gcd(r_0, r_1)=1\\\\). we obtain\n\\\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\\\]\ntaking this equation modulo \\\\(r_0\\\\) we obtain\n\\\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\n\\\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\nt is the definition of the inverse of \\\\(r_1\\\\)\n\nThus, if we need to compute an inverse \\\\(a^{-1} \\bmod m\\\\), we apply EEA with the input parameters \\\\(m\\\\) and \\\\(a\\\\)\n## Euler's Phi Function\nwe consider the ring \\\\( Z_m\\\\) i.e., the set of integers \\\\({0,1,...,m-1}\\\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler's phi function, which is \\\\(\\Phi(m)\\\\)\n\n***\nlet m have the following canonical factorization\n\\\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot ... \\cdot p_{n}^{e_n}\\\\]\nwhere the \\\\(p_i\\\\) are distinct prime numbers and \\\\( e_i\\\\) are positive integers, then\n\n\\\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\\\]\n***\nit is important to stress that we need to know the factoorization of m in order to calculate Euler's phi function.\n\n## Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\nthe theorem can be stated in the form also,\n\\\\[ a^{p-1} \\equiv 1 \\bmod p\\\\]\nthen the inverse of an integer is,\n\\\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\\\]\nperforming the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat's Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.\n\na generatlization of Fermat's little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler's theorem.\n***\n**Euler's Theorem**\nlet \\\\(a\\\\) and \\\\(m\\\\) be integers with \\\\(gcd(a,m) = 1\\\\), then\n\\\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\\\]\n***\nsince it works modulo m, it is applicable to integer rings \\\\(Z_{m}\\\\)\n\n## key generation\n***\n**Output**: public key: \\\\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\\\)\n1. choose two large primes p and q.\n2. compute \\\\(n = p\\cdot q\\\\)\n3. compute \\\\( \\Phi(n) = (p-1)(q-1)\\\\)\n4. select the public exponent \\\\( e \\in {1,2,...,\\Phi(n)-1} \\\\) such that \n\\\\[ gcd(e,\\Phi(n)) = 1\\\\]\n5. compute the private key d such that\n\\\\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\\\]\n***\nthe condition that \\\\( gcd(e,\\Phi(n)) = 1\\\\) ensures that the inverse of \\\\(e\\\\) exists modulo \\\\(\\Phi(n)\\\\), so that there is always a private key \\\\(d\\\\).\nthe computation of key keys \\\\(d\\\\) and \\\\(e\\\\) canb e doen at once using the extended Euclidean algorith. \n\n\n## Encryption and Decryption\n\n***\n**RSA Encryption** Given the privaate key \\\\( k_{pub} = (n,e) \\\\) and the plaintext \\\\(x\\\\), the encryption is:\n\\\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n***\n**RSA Decryption** Given the public key \\\\d = k_{pr} \\\\) and the plaintext \\\\(y\\\\), the decryption is:\n\\\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n\n## Digital signature\nthe message \\\\(x\\\\) that is being signed is in the range \\\\(1,2,...,n-1\\\\)\n![rsa digital signature](/images/cryptography/rsa/rsa_signature.png)\n## references\n- [1] [A Method for Obtaining Digital\nSignatures and Public-Key Cryptosystems](https://web.williams.edu/Mathematics/lg5/302/RSA.pdf) ","source":"_posts/cryptography/cryptography-02-rsa.md","raw":"---\ntitle: cryptography (2) RSA cryptosystem\ndate: 2023-06-10 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n## introduction\nthe security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].\n\n## Euclidean Algorithm\nthe gcd of two positive integers \\\\(r_0\\\\) and \\\\(r_1\\\\) is denoted by \n\\\\[gcd(r_0, r_1)\\\\]\nthe Eucliedean algorithm is used to calculate gcd, based on as simple observation that\n\\\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\\\]\nwhere we assume \\\\(r_0 > r_1\\\\)\nThis property can easily be proven: Let \\\\(gcd(r_0, r_1) = g\\\\), since \\\\(g\\\\) divides both \\\\(r_0\\\\) and \\\\(r_1\\\\), we can write \\\\(r_0 = g \\cdot x\\\\) and \\\\(r_1 = g \\cdot y\\\\), where \\\\(x>y\\\\) and \\\\(x\\\\) and \\\\(y\\\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\\\((x-y)\\\\) and \\\\(y\\\\) are also coprime. it follows from here that\n\\\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\\\]\n\nit also follow immediately that we can apply the process iteratively:\n\\\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\\\]\nas long as \\\\((r_0 - mr_1) >0\\\\). the algorithm use the fewest number of steps if we choose maximum value of \\\\(m\\\\). this is the case if we compute:\n\\\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\\\]\nthis process can be applied recursively until we obtain finally \\\\[gcd(r_l, 0) = r_l \\\\]\nthe euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.\n## Extended Euclidean Algorithm\nan extension of the Euclidean algorithm allows us to compute ***modular inverse***. in addition to computing the gcd, the ***extended Euclidean algorithm*** computes a linear combination of the form\n\\\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\\\]\nwhere s and t are integer coefficients. this equation is ofthen referred to as ***Diophantine equation***\n\nthe detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.\nlet \\\\(r_0 =973 , r_1 = 301\\\\). during the steps of Euclidean Algorithm, we obtain \\\\(973 = 3\\cdot301 + 70\\\\)\nwhich is \\\\[r_0 = q_1 \\cdot r_1 + r_2\\\\] \nrearrange:\n\\\\[r_2 = r_0 + (-q_1) \\cdot r_1\\\\] \nreplacing (r_0, r_1) and iteratively by (r_1, r_2), ... (r_{i-1}, r_{i}), util \\\\(r_{i+1} = 0\\\\)\nthen \\\\(r_{i}\\\\) is \\\\(gcd(r_0,r_1)\\\\), and can have a representation of \n\\\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\\\]. \nsince the inverse only exists if \\\\(gcd(r_0, r_1)=1\\\\). we obtain\n\\\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\\\]\ntaking this equation modulo \\\\(r_0\\\\) we obtain\n\\\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\n\\\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\nt is the definition of the inverse of \\\\(r_1\\\\)\n\nThus, if we need to compute an inverse \\\\(a^{-1} \\bmod m\\\\), we apply EEA with the input parameters \\\\(m\\\\) and \\\\(a\\\\)\n## Euler's Phi Function\nwe consider the ring \\\\( Z_m\\\\) i.e., the set of integers \\\\({0,1,...,m-1}\\\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler's phi function, which is \\\\(\\Phi(m)\\\\)\n\n***\nlet m have the following canonical factorization\n\\\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot ... \\cdot p_{n}^{e_n}\\\\]\nwhere the \\\\(p_i\\\\) are distinct prime numbers and \\\\( e_i\\\\) are positive integers, then\n\n\\\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\\\]\n***\nit is important to stress that we need to know the factoorization of m in order to calculate Euler's phi function.\n\n## Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\nthe theorem can be stated in the form also,\n\\\\[ a^{p-1} \\equiv 1 \\bmod p\\\\]\nthen the inverse of an integer is,\n\\\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\\\]\nperforming the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat's Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.\n\na generatlization of Fermat's little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler's theorem.\n***\n**Euler's Theorem**\nlet \\\\(a\\\\) and \\\\(m\\\\) be integers with \\\\(gcd(a,m) = 1\\\\), then\n\\\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\\\]\n***\nsince it works modulo m, it is applicable to integer rings \\\\(Z_{m}\\\\)\n\n## key generation\n***\n**Output**: public key: \\\\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\\\)\n1. choose two large primes p and q.\n2. compute \\\\(n = p\\cdot q\\\\)\n3. compute \\\\( \\Phi(n) = (p-1)(q-1)\\\\)\n4. select the public exponent \\\\( e \\in {1,2,...,\\Phi(n)-1} \\\\) such that \n\\\\[ gcd(e,\\Phi(n)) = 1\\\\]\n5. compute the private key d such that\n\\\\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\\\]\n***\nthe condition that \\\\( gcd(e,\\Phi(n)) = 1\\\\) ensures that the inverse of \\\\(e\\\\) exists modulo \\\\(\\Phi(n)\\\\), so that there is always a private key \\\\(d\\\\).\nthe computation of key keys \\\\(d\\\\) and \\\\(e\\\\) canb e doen at once using the extended Euclidean algorith. \n\n\n## Encryption and Decryption\n\n***\n**RSA Encryption** Given the privaate key \\\\( k_{pub} = (n,e) \\\\) and the plaintext \\\\(x\\\\), the encryption is:\n\\\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n***\n**RSA Decryption** Given the public key \\\\d = k_{pr} \\\\) and the plaintext \\\\(y\\\\), the decryption is:\n\\\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n\n## Digital signature\nthe message \\\\(x\\\\) that is being signed is in the range \\\\(1,2,...,n-1\\\\)\n![rsa digital signature](/images/cryptography/rsa/rsa_signature.png)\n## references\n- [1] [A Method for Obtaining Digital\nSignatures and Public-Key Cryptosystems](https://web.williams.edu/Mathematics/lg5/302/RSA.pdf) ","slug":"cryptography/cryptography-02-rsa","published":1,"updated":"2023-07-15T06:54:24.042Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzl0007ofsj05l8frvy","content":"\n\n\n\n

introduction

the security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].

\n

Euclidean Algorithm

the gcd of two positive integers \\(r_0\\) and \\(r_1\\) is denoted by
\\[gcd(r_0, r_1)\\]
the Eucliedean algorithm is used to calculate gcd, based on as simple observation that
\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\]
where we assume \\(r_0 > r_1\\)
This property can easily be proven: Let \\(gcd(r_0, r_1) = g\\), since \\(g\\) divides both \\(r_0\\) and \\(r_1\\), we can write \\(r_0 = g \\cdot x\\) and \\(r_1 = g \\cdot y\\), where \\(x>y\\) and \\(x\\) and \\(y\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\((x-y)\\) and \\(y\\) are also coprime. it follows from here that
\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\]

\n

it also follow immediately that we can apply the process iteratively:
\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\]
as long as \\((r_0 - mr_1) >0\\). the algorithm use the fewest number of steps if we choose maximum value of \\(m\\). this is the case if we compute:
\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\]
this process can be applied recursively until we obtain finally \\[gcd(r_l, 0) = r_l \\]
the euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.

\n

Extended Euclidean Algorithm

an extension of the Euclidean algorithm allows us to compute modular inverse. in addition to computing the gcd, the extended Euclidean algorithm computes a linear combination of the form
\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\]
where s and t are integer coefficients. this equation is ofthen referred to as Diophantine equation

\n

the detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.
let \\(r_0 =973 , r_1 = 301\\). during the steps of Euclidean Algorithm, we obtain \\(973 = 3\\cdot301 + 70\\)
which is \\[r_0 = q_1 \\cdot r_1 + r_2\\]
rearrange:
\\[r_2 = r_0 + (-q_1) \\cdot r_1\\]
replacing (r_0, r_1) and iteratively by (r_1, r_2), … (r_{i-1}, r_{i}), util \\(r_{i+1} = 0\\)
then \\(r_{i}\\) is \\(gcd(r_0,r_1)\\), and can have a representation of
\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\].
since the inverse only exists if \\(gcd(r_0, r_1)=1\\). we obtain
\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\]
taking this equation modulo \\(r_0\\) we obtain
\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
t is the definition of the inverse of \\(r_1\\)

\n

Thus, if we need to compute an inverse \\(a^{-1} \\bmod m\\), we apply EEA with the input parameters \\(m\\) and \\(a\\)

\n

Euler’s Phi Function

we consider the ring \\( Z_m\\) i.e., the set of integers \\({0,1,…,m-1}\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler’s phi function, which is \\(\\Phi(m)\\)

\n
\n

let m have the following canonical factorization
\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot … \\cdot p_{n}^{e_n}\\]
where the \\(p_i\\) are distinct prime numbers and \\( e_i\\) are positive integers, then

\n

\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\]

\n
\n

it is important to stress that we need to know the factoorization of m in order to calculate Euler’s phi function.

\n

Fermat’s little theorem

Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
\\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
\\[ a^{p} \\equiv a \\bmod p\\]
the theorem can be stated in the form also,
\\[ a^{p-1} \\equiv 1 \\bmod p\\]
then the inverse of an integer is,
\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\]
performing the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat’s Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.

\n

a generatlization of Fermat’s little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler’s theorem.

\n
\n

Euler’s Theorem
let \\(a\\) and \\(m\\) be integers with \\(gcd(a,m) = 1\\), then
\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\]

\n
\n

since it works modulo m, it is applicable to integer rings \\(Z_{m}\\)

\n

key generation

\n

Output: public key: \\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\)

\n
    \n
  1. choose two large primes p and q.
    2. \n
  2. compute \\(n = p\\cdot q\\)
    3. \n
  3. compute \\( \\Phi(n) = (p-1)(q-1)\\)
    4. \n
  4. select the public exponent \\( e \\in {1,2,…,\\Phi(n)-1} \\) such that
    \\[ gcd(e,\\Phi(n)) = 1\\]
    5. \n
  5. compute the private key d such that
    \\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\]
    6. \n
\n
\n

the condition that \\( gcd(e,\\Phi(n)) = 1\\) ensures that the inverse of \\(e\\) exists modulo \\(\\Phi(n)\\), so that there is always a private key \\(d\\).
the computation of key keys \\(d\\) and \\(e\\) canb e doen at once using the extended Euclidean algorith.

\n

Encryption and Decryption

\n

RSA Encryption Given the privaate key \\( k_{pub} = (n,e) \\) and the plaintext \\(x\\), the encryption is:
\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n
\n

RSA Decryption Given the public key \\d = k_{pr} \\) and the plaintext \\(y\\), the decryption is:
\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n

Digital signature

the message \\(x\\) that is being signed is in the range \\(1,2,…,n-1\\)
\"rsa

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

introduction

the security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].

\n

Euclidean Algorithm

the gcd of two positive integers \\(r_0\\) and \\(r_1\\) is denoted by
\\[gcd(r_0, r_1)\\]
the Eucliedean algorithm is used to calculate gcd, based on as simple observation that
\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\]
where we assume \\(r_0 > r_1\\)
This property can easily be proven: Let \\(gcd(r_0, r_1) = g\\), since \\(g\\) divides both \\(r_0\\) and \\(r_1\\), we can write \\(r_0 = g \\cdot x\\) and \\(r_1 = g \\cdot y\\), where \\(x>y\\) and \\(x\\) and \\(y\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\((x-y)\\) and \\(y\\) are also coprime. it follows from here that
\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\]

\n

it also follow immediately that we can apply the process iteratively:
\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\]
as long as \\((r_0 - mr_1) >0\\). the algorithm use the fewest number of steps if we choose maximum value of \\(m\\). this is the case if we compute:
\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\]
this process can be applied recursively until we obtain finally \\[gcd(r_l, 0) = r_l \\]
the euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.

\n

Extended Euclidean Algorithm

an extension of the Euclidean algorithm allows us to compute modular inverse. in addition to computing the gcd, the extended Euclidean algorithm computes a linear combination of the form
\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\]
where s and t are integer coefficients. this equation is ofthen referred to as Diophantine equation

\n

the detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.
let \\(r_0 =973 , r_1 = 301\\). during the steps of Euclidean Algorithm, we obtain \\(973 = 3\\cdot301 + 70\\)
which is \\[r_0 = q_1 \\cdot r_1 + r_2\\]
rearrange:
\\[r_2 = r_0 + (-q_1) \\cdot r_1\\]
replacing (r_0, r_1) and iteratively by (r_1, r_2), … (r_{i-1}, r_{i}), util \\(r_{i+1} = 0\\)
then \\(r_{i}\\) is \\(gcd(r_0,r_1)\\), and can have a representation of
\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\].
since the inverse only exists if \\(gcd(r_0, r_1)=1\\). we obtain
\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\]
taking this equation modulo \\(r_0\\) we obtain
\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
t is the definition of the inverse of \\(r_1\\)

\n

Thus, if we need to compute an inverse \\(a^{-1} \\bmod m\\), we apply EEA with the input parameters \\(m\\) and \\(a\\)

\n

Euler’s Phi Function

we consider the ring \\( Z_m\\) i.e., the set of integers \\({0,1,…,m-1}\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler’s phi function, which is \\(\\Phi(m)\\)

\n
\n

let m have the following canonical factorization
\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot … \\cdot p_{n}^{e_n}\\]
where the \\(p_i\\) are distinct prime numbers and \\( e_i\\) are positive integers, then

\n

\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\]

\n
\n

it is important to stress that we need to know the factoorization of m in order to calculate Euler’s phi function.

\n

Fermat’s little theorem

Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
\\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
\\[ a^{p} \\equiv a \\bmod p\\]
the theorem can be stated in the form also,
\\[ a^{p-1} \\equiv 1 \\bmod p\\]
then the inverse of an integer is,
\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\]
performing the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat’s Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.

\n

a generatlization of Fermat’s little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler’s theorem.

\n
\n

Euler’s Theorem
let \\(a\\) and \\(m\\) be integers with \\(gcd(a,m) = 1\\), then
\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\]

\n
\n

since it works modulo m, it is applicable to integer rings \\(Z_{m}\\)

\n

key generation

\n

Output: public key: \\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\)

\n
    \n
  1. choose two large primes p and q.
    2. \n
  2. compute \\(n = p\\cdot q\\)
    3. \n
  3. compute \\( \\Phi(n) = (p-1)(q-1)\\)
    4. \n
  4. select the public exponent \\( e \\in {1,2,…,\\Phi(n)-1} \\) such that
    \\[ gcd(e,\\Phi(n)) = 1\\]
    5. \n
  5. compute the private key d such that
    \\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\]
    6. \n
\n
\n

the condition that \\( gcd(e,\\Phi(n)) = 1\\) ensures that the inverse of \\(e\\) exists modulo \\(\\Phi(n)\\), so that there is always a private key \\(d\\).
the computation of key keys \\(d\\) and \\(e\\) canb e doen at once using the extended Euclidean algorith.

\n

Encryption and Decryption

\n

RSA Encryption Given the privaate key \\( k_{pub} = (n,e) \\) and the plaintext \\(x\\), the encryption is:
\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n
\n

RSA Decryption Given the public key \\d = k_{pr} \\) and the plaintext \\(y\\), the decryption is:
\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n

Digital signature

the message \\(x\\) that is being signed is in the range \\(1,2,…,n-1\\)
\"rsa

\n

references

\n"},{"title":"cryptography (3) elliptic curve","date":"2023-06-17T06:29:26.000Z","_content":"\n\n\n\n## elliptic curve definition\nfor cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields `GF(p)`, where all arithmetic is performed modulo a prime p.\n***\nthe elliptic curve over \\\\( Z_{p}, p>3 \\\\), is the set of all pairs \\\\( (x,y) \\in Z_{p} \\\\) which fulfill\n \\\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\\\]\ntogether with an imaginary point of infinity \\\\( \\mathcal{O} \\\\), where\n \\\\[ a,b \\in Z_{p} \\\\]\n and the condition \\\\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\\\)\n***\nthe definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\\\( -16(4a^3) + 27b^2 \\\\) is nonzero.\n\n## operations on elliptic curve\n![point addition](/images/cryptography/elliptic_curve/point_addition.webp)\nlet's denote the group operation with the addition symbol `+`. \"addition\" means that given two points and their coordinates, say \\\\( P = (x_1, y_1) \\\\) and \\\\( Q = (x_2, y_2) \\\\), we have to compute the coordidnate of a third point \\\\( R (x_3, y_3) \\\\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling `(P+P = 2P)` is a tangent line through the point P, and the rest is same.\nthe fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below \n***\n \\\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\\\]\n \\\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\\\]\n where \n\\begin{equation}\ns = \n\\begin{cases}\n\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr\n\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})\n\\end{cases}\n\\end{equation}\n***\nnote that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.\nFurthmore, we define an abstract point at infinity as the neutral element \\\\( \\mathcal{O} \\\\). \n\\\\[ P + \\mathcal{O} = P\\\\]\naccording the group definition, we can now also define the inverse `-P` of any group element P as\n\\\\[ P + (-P) = \\mathcal{O}\\\\]\nTherefore, the inverse of \\\\( P = (x_p, y_p) \\\\) is just \\\\( P = (x_p, -y_p)\\\\). since \\\\( -y_p \\equiv p - y_p\\\\), hence\n\\\\[ -P = (x_p, p-y_p) \\\\]\n\n## DLP(discrete log problem) with Elliptic curve\nto set up DL cryptosystems it is important to know the order of the group.\nHasse's theorem\n***\ngiven an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:\n\\\\[ p+1-2\\sqrt{p} \\leq \\\\#E \\leq p+1+2\\sqrt{p} \\\\]\n***\nElliptic Curved Discrete Logarithm Problem (ECDLP)\n***\nGiven an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\\\( 1 \\leq d \\leq \\\\#E \\\\), such that:\n\\\\[ P + P + ....+ P = dP = T \\\\]\n***\nIn cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\\\( T = (x_T, y_T)\\\\)\nUsually a **square-and-multiply** algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post). \n","source":"_posts/cryptography/cryptography-03-elliptic-curve.md","raw":"---\ntitle: cryptography (3) elliptic curve\ndate: 2023-06-17 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n## elliptic curve definition\nfor cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields `GF(p)`, where all arithmetic is performed modulo a prime p.\n***\nthe elliptic curve over \\\\( Z_{p}, p>3 \\\\), is the set of all pairs \\\\( (x,y) \\in Z_{p} \\\\) which fulfill\n \\\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\\\]\ntogether with an imaginary point of infinity \\\\( \\mathcal{O} \\\\), where\n \\\\[ a,b \\in Z_{p} \\\\]\n and the condition \\\\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\\\)\n***\nthe definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\\\( -16(4a^3) + 27b^2 \\\\) is nonzero.\n\n## operations on elliptic curve\n![point addition](/images/cryptography/elliptic_curve/point_addition.webp)\nlet's denote the group operation with the addition symbol `+`. \"addition\" means that given two points and their coordinates, say \\\\( P = (x_1, y_1) \\\\) and \\\\( Q = (x_2, y_2) \\\\), we have to compute the coordidnate of a third point \\\\( R (x_3, y_3) \\\\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling `(P+P = 2P)` is a tangent line through the point P, and the rest is same.\nthe fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below \n***\n \\\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\\\]\n \\\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\\\]\n where \n\\begin{equation}\ns = \n\\begin{cases}\n\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr\n\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})\n\\end{cases}\n\\end{equation}\n***\nnote that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.\nFurthmore, we define an abstract point at infinity as the neutral element \\\\( \\mathcal{O} \\\\). \n\\\\[ P + \\mathcal{O} = P\\\\]\naccording the group definition, we can now also define the inverse `-P` of any group element P as\n\\\\[ P + (-P) = \\mathcal{O}\\\\]\nTherefore, the inverse of \\\\( P = (x_p, y_p) \\\\) is just \\\\( P = (x_p, -y_p)\\\\). since \\\\( -y_p \\equiv p - y_p\\\\), hence\n\\\\[ -P = (x_p, p-y_p) \\\\]\n\n## DLP(discrete log problem) with Elliptic curve\nto set up DL cryptosystems it is important to know the order of the group.\nHasse's theorem\n***\ngiven an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:\n\\\\[ p+1-2\\sqrt{p} \\leq \\\\#E \\leq p+1+2\\sqrt{p} \\\\]\n***\nElliptic Curved Discrete Logarithm Problem (ECDLP)\n***\nGiven an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\\\( 1 \\leq d \\leq \\\\#E \\\\), such that:\n\\\\[ P + P + ....+ P = dP = T \\\\]\n***\nIn cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\\\( T = (x_T, y_T)\\\\)\nUsually a **square-and-multiply** algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post). \n","slug":"cryptography/cryptography-03-elliptic-curve","published":1,"updated":"2023-07-15T06:52:53.426Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzl0008ofsjc9z43usj","content":"\n\n\n\n

elliptic curve definition

for cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields GF(p), where all arithmetic is performed modulo a prime p.

\n
\n

the elliptic curve over \\( Z_{p}, p>3 \\), is the set of all pairs \\( (x,y) \\in Z_{p} \\) which fulfill
\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\]
together with an imaginary point of infinity \\( \\mathcal{O} \\), where
\\[ a,b \\in Z_{p} \\]
and the condition \\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\)

\n
\n

the definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\( -16(4a^3) + 27b^2 \\) is nonzero.

\n

operations on elliptic curve

\"point
let’s denote the group operation with the addition symbol +. “addition” means that given two points and their coordinates, say \\( P = (x_1, y_1) \\) and \\( Q = (x_2, y_2) \\), we have to compute the coordidnate of a third point \\( R (x_3, y_3) \\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling (P+P = 2P) is a tangent line through the point P, and the rest is same.
the fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below

\n
\n

\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\]
\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\]
where
\\begin{equation}
s =
\\begin{cases}
\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr
\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})
\\end{cases}
\\end{equation}

\n
\n

note that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.
Furthmore, we define an abstract point at infinity as the neutral element \\( \\mathcal{O} \\).
\\[ P + \\mathcal{O} = P\\]
according the group definition, we can now also define the inverse -P of any group element P as
\\[ P + (-P) = \\mathcal{O}\\]
Therefore, the inverse of \\( P = (x_p, y_p) \\) is just \\( P = (x_p, -y_p)\\). since \\( -y_p \\equiv p - y_p\\), hence
\\[ -P = (x_p, p-y_p) \\]

\n

DLP(discrete log problem) with Elliptic curve

to set up DL cryptosystems it is important to know the order of the group.
Hasse’s theorem

\n
\n

given an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:
\\[ p+1-2\\sqrt{p} \\leq \\#E \\leq p+1+2\\sqrt{p} \\]

\n
\n

Elliptic Curved Discrete Logarithm Problem (ECDLP)

\n
\n

Given an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\( 1 \\leq d \\leq \\#E \\), such that:
\\[ P + P + ….+ P = dP = T \\]

\n
\n

In cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\( T = (x_T, y_T)\\)
Usually a square-and-multiply algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post).

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

elliptic curve definition

for cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields GF(p), where all arithmetic is performed modulo a prime p.

\n
\n

the elliptic curve over \\( Z_{p}, p>3 \\), is the set of all pairs \\( (x,y) \\in Z_{p} \\) which fulfill
\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\]
together with an imaginary point of infinity \\( \\mathcal{O} \\), where
\\[ a,b \\in Z_{p} \\]
and the condition \\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\)

\n
\n

the definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\( -16(4a^3) + 27b^2 \\) is nonzero.

\n

operations on elliptic curve

\"point
let’s denote the group operation with the addition symbol +. “addition” means that given two points and their coordinates, say \\( P = (x_1, y_1) \\) and \\( Q = (x_2, y_2) \\), we have to compute the coordidnate of a third point \\( R (x_3, y_3) \\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling (P+P = 2P) is a tangent line through the point P, and the rest is same.
the fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below

\n
\n

\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\]
\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\]
where
\\begin{equation}
s =
\\begin{cases}
\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr
\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})
\\end{cases}
\\end{equation}

\n
\n

note that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.
Furthmore, we define an abstract point at infinity as the neutral element \\( \\mathcal{O} \\).
\\[ P + \\mathcal{O} = P\\]
according the group definition, we can now also define the inverse -P of any group element P as
\\[ P + (-P) = \\mathcal{O}\\]
Therefore, the inverse of \\( P = (x_p, y_p) \\) is just \\( P = (x_p, -y_p)\\). since \\( -y_p \\equiv p - y_p\\), hence
\\[ -P = (x_p, p-y_p) \\]

\n

DLP(discrete log problem) with Elliptic curve

to set up DL cryptosystems it is important to know the order of the group.
Hasse’s theorem

\n
\n

given an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:
\\[ p+1-2\\sqrt{p} \\leq \\#E \\leq p+1+2\\sqrt{p} \\]

\n
\n

Elliptic Curved Discrete Logarithm Problem (ECDLP)

\n
\n

Given an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\( 1 \\leq d \\leq \\#E \\), such that:
\\[ P + P + ….+ P = dP = T \\]

\n
\n

In cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\( T = (x_T, y_T)\\)
Usually a square-and-multiply algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post).

\n"},{"title":"cryptography (4) digital signature","date":"2023-06-20T06:29:26.000Z","_content":"\n\n\n## Elgamal Digital Signature Scheme\nThe Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.\n\n### key generation\n***\n1. Choose a large prime \\\\(p\\\\).\n2. Choose a primitive element \\\\(\\alpha\\\\) of \\\\(Z_{p}^{\\ast}\\\\), or a subgroup of \\\\(Z_{p}^{\\ast}\\\\).\n3. Choose a random integer \\\\(d \\in {2,3,...,p-2}\\\\)\n4. Compute \\\\(\\beta = \\alpha^{d} \\bmod p\\\\)\n***\nThe public key is now formed by \\\\(k_{pub} = (p, \\alpha, \\beta)\\\\), and the private key by \\\\(k_{pr}=d\\\\)\n\\\\(Z_{p}^{\\ast}\\\\) is the set of integers who are smaller than \\\\(p\\\\) and coprime to \\\\(p\\\\)\n\n### signature and verification\nUsign the private ey and parameters of the public key, the signature\n\\\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\\\]\n\\\\(x\\\\) is the message. \\\\(k_{E}\\\\) is a random value, which forms an ephemeral private key\n***\n**Elgamal Signature Generation**\n1. choose a random ephemeral key \\\\(k_{E} \\in {0,1,2,..,p-2}\\\\) such that \\\\(gcd(k_{E}, p-1) = 1\\\\)\n2. compute the signatue parameters:\n\\\\[r \\equiv \\alpha^{k_{E}} \\bmod p\\\\]\n\\\\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\\\]\n***\non the receiving side, the signature is verified as \\\\(ver_{k_{pub}}(x,(r,s))\\\\) using the public key, the signature and the message.\n***\n**Elgamal Signature Verification**\n1. comput the value \n\\\\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\\\]\n2. the verification follows form\n\\begin{equation}\nt = \n\\begin{cases}\n\\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr\n\\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Digital Signature Algorithm (DSA)\nThe native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks\n that can threaten the Elgamal scheme are not applicable.\n### key generation\n***\n**key Generation for DSA**\n1. Generate a prime \\\\(p\\\\) with \\\\(2^1023 < p < 2^1024\\\\)\n2. find a prime divisor \\\\(q\\\\) of \\\\(p-1\\\\) \\\\(2^159 < q < 2^160\\\\)\n3. Find an element \\\\(\\alpha\\\\) with \\\\( ord(\\alpha) = q\\\\), i.e., \\alpha genertes the subgroup with \\\\(q\\\\) elements.\n4. choose a random integer \\\\(d\\\\) with \\\\(0 < d < q\\\\).\n5. compute \\\\(\\beta \\equiv \\alpha^{d} \\bmod p\\\\).\nthe keys are now:\n\\\\(k_{pub} = (p, q, \\alpha, \\beta)\\\\)\n\\\\(k_{pr}= (d)\\\\)\n***\nThe central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\\\(Z_{p}*{\\ast}\\\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\\\(Z_{p}^{\\ast}\\\\). this set-up yields shorter signature.\n\n### Signature and Verification\nAs in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\\\((r,s)\\\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit. \n***\n**DSA signature generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\(0 < k_{E} < q\\\\).\n2. compute \\\\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\\\)\n3. compute \\\\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\\\)\n***\n\nThe signature verification process is as follows:\n***\n**DSA signature verification**\n1. compute auxilary value \\\\(w \\equiv s^{-1} \\bmod q\\\\).\n2. compute auxilary value \\\\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\\\).\n3. compute auxilary value \\\\(u_{2} \\equiv w \\cdot r \\bmod q\\\\).\n4. compute \\\\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\\\).\n5. the verification \\\\(ver_{k_{pub}}(x, (r,s))\\\\) folows from\n\\begin{equation}\nv = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Elliptic Curve Digital Signature Algorithm (ECDSA)\nElliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures. \nThe ECDSA standard is defined for elliptic curves over prime fields \\\\(Z_{p}\\\\) adn Galois fields \\\\(GF(2^m)\\\\). the former is often preferred in practice, and we only introduce this one in what follows\n### key generation\n***\n**Key Generation for ECDSA**\n1. Use and elliptic curve E with \n- modulus p\n- coefficients a and b\n- a point A which generates a cyclic group of prime order q.\n2. choose a random integer d with \\\\(0 < d < q\\\\)\n3. compute \\\\(B = dA\\\\).\nThe keys are now\n\\\\(k_{pub} = (p,a,b,q,A,B)\\\\)\n\\\\(k_{pr} = (d)\\\\)\n***\n\n### Signature and Verification\n***\n**ECDSA Signature Generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\( 0 < k_{E} < q\\\\).\n2. compute \\\\(R = k_{E}A\\\\)\n3. Let \\\\(r = x_{R}\\\\)\n4. compute \\\\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\\\)\n***\nthe signature verification process is as follows\n***\n**ECDSA Signature Verification**\n1. Compute auxiliary value \\\\(w \\equiv s^{-1} \\bmod q\\\\)\n2. compute auxilary value \\\\(u_1 \\equiv w \\cdot h(x) \\bmod q\\\\)\n3. compute auxiliary value \\\\(u_2 = w \\cdot r \\bmod q\\\\)\n4. compute \\\\(P = u_1 A + u_2 B\\\\).\n5. the verification \\\\(ver{k_{pub}}(x, (r,s))\\\\) follows from\n\\begin{equation}\nx_{P} = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\nThe point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.","source":"_posts/cryptography/cryptography-04-digital-signature.md","raw":"---\ntitle: cryptography (4) digital signature\ndate: 2023-06-20 14:29:26\ntags: [cryptography]\n---\n\n\n\n## Elgamal Digital Signature Scheme\nThe Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.\n\n### key generation\n***\n1. Choose a large prime \\\\(p\\\\).\n2. Choose a primitive element \\\\(\\alpha\\\\) of \\\\(Z_{p}^{\\ast}\\\\), or a subgroup of \\\\(Z_{p}^{\\ast}\\\\).\n3. Choose a random integer \\\\(d \\in {2,3,...,p-2}\\\\)\n4. Compute \\\\(\\beta = \\alpha^{d} \\bmod p\\\\)\n***\nThe public key is now formed by \\\\(k_{pub} = (p, \\alpha, \\beta)\\\\), and the private key by \\\\(k_{pr}=d\\\\)\n\\\\(Z_{p}^{\\ast}\\\\) is the set of integers who are smaller than \\\\(p\\\\) and coprime to \\\\(p\\\\)\n\n### signature and verification\nUsign the private ey and parameters of the public key, the signature\n\\\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\\\]\n\\\\(x\\\\) is the message. \\\\(k_{E}\\\\) is a random value, which forms an ephemeral private key\n***\n**Elgamal Signature Generation**\n1. choose a random ephemeral key \\\\(k_{E} \\in {0,1,2,..,p-2}\\\\) such that \\\\(gcd(k_{E}, p-1) = 1\\\\)\n2. compute the signatue parameters:\n\\\\[r \\equiv \\alpha^{k_{E}} \\bmod p\\\\]\n\\\\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\\\]\n***\non the receiving side, the signature is verified as \\\\(ver_{k_{pub}}(x,(r,s))\\\\) using the public key, the signature and the message.\n***\n**Elgamal Signature Verification**\n1. comput the value \n\\\\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\\\]\n2. the verification follows form\n\\begin{equation}\nt = \n\\begin{cases}\n\\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr\n\\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Digital Signature Algorithm (DSA)\nThe native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks\n that can threaten the Elgamal scheme are not applicable.\n### key generation\n***\n**key Generation for DSA**\n1. Generate a prime \\\\(p\\\\) with \\\\(2^1023 < p < 2^1024\\\\)\n2. find a prime divisor \\\\(q\\\\) of \\\\(p-1\\\\) \\\\(2^159 < q < 2^160\\\\)\n3. Find an element \\\\(\\alpha\\\\) with \\\\( ord(\\alpha) = q\\\\), i.e., \\alpha genertes the subgroup with \\\\(q\\\\) elements.\n4. choose a random integer \\\\(d\\\\) with \\\\(0 < d < q\\\\).\n5. compute \\\\(\\beta \\equiv \\alpha^{d} \\bmod p\\\\).\nthe keys are now:\n\\\\(k_{pub} = (p, q, \\alpha, \\beta)\\\\)\n\\\\(k_{pr}= (d)\\\\)\n***\nThe central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\\\(Z_{p}*{\\ast}\\\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\\\(Z_{p}^{\\ast}\\\\). this set-up yields shorter signature.\n\n### Signature and Verification\nAs in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\\\((r,s)\\\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit. \n***\n**DSA signature generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\(0 < k_{E} < q\\\\).\n2. compute \\\\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\\\)\n3. compute \\\\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\\\)\n***\n\nThe signature verification process is as follows:\n***\n**DSA signature verification**\n1. compute auxilary value \\\\(w \\equiv s^{-1} \\bmod q\\\\).\n2. compute auxilary value \\\\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\\\).\n3. compute auxilary value \\\\(u_{2} \\equiv w \\cdot r \\bmod q\\\\).\n4. compute \\\\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\\\).\n5. the verification \\\\(ver_{k_{pub}}(x, (r,s))\\\\) folows from\n\\begin{equation}\nv = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Elliptic Curve Digital Signature Algorithm (ECDSA)\nElliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures. \nThe ECDSA standard is defined for elliptic curves over prime fields \\\\(Z_{p}\\\\) adn Galois fields \\\\(GF(2^m)\\\\). the former is often preferred in practice, and we only introduce this one in what follows\n### key generation\n***\n**Key Generation for ECDSA**\n1. Use and elliptic curve E with \n- modulus p\n- coefficients a and b\n- a point A which generates a cyclic group of prime order q.\n2. choose a random integer d with \\\\(0 < d < q\\\\)\n3. compute \\\\(B = dA\\\\).\nThe keys are now\n\\\\(k_{pub} = (p,a,b,q,A,B)\\\\)\n\\\\(k_{pr} = (d)\\\\)\n***\n\n### Signature and Verification\n***\n**ECDSA Signature Generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\( 0 < k_{E} < q\\\\).\n2. compute \\\\(R = k_{E}A\\\\)\n3. Let \\\\(r = x_{R}\\\\)\n4. compute \\\\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\\\)\n***\nthe signature verification process is as follows\n***\n**ECDSA Signature Verification**\n1. Compute auxiliary value \\\\(w \\equiv s^{-1} \\bmod q\\\\)\n2. compute auxilary value \\\\(u_1 \\equiv w \\cdot h(x) \\bmod q\\\\)\n3. compute auxiliary value \\\\(u_2 = w \\cdot r \\bmod q\\\\)\n4. compute \\\\(P = u_1 A + u_2 B\\\\).\n5. the verification \\\\(ver{k_{pub}}(x, (r,s))\\\\) follows from\n\\begin{equation}\nx_{P} = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\nThe point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.","slug":"cryptography/cryptography-04-digital-signature","published":1,"updated":"2023-07-16T02:00:39.727Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzm000aofsj62am2966","content":"\n\n\n

Elgamal Digital Signature Scheme

The Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.

\n

key generation

\n
    \n
  1. Choose a large prime \\(p\\).
    2. \n
  2. Choose a primitive element \\(\\alpha\\) of \\(Z_{p}^{\\ast}\\), or a subgroup of \\(Z_{p}^{\\ast}\\).
    3. \n
  3. Choose a random integer \\(d \\in {2,3,…,p-2}\\)
    4. \n
  4. Compute \\(\\beta = \\alpha^{d} \\bmod p\\)
    5. \n
\n
\n

The public key is now formed by \\(k_{pub} = (p, \\alpha, \\beta)\\), and the private key by \\(k_{pr}=d\\)
\\(Z_{p}^{\\ast}\\) is the set of integers who are smaller than \\(p\\) and coprime to \\(p\\)

\n

signature and verification

Usign the private ey and parameters of the public key, the signature
\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\]
\\(x\\) is the message. \\(k_{E}\\) is a random value, which forms an ephemeral private key

\n
\n

Elgamal Signature Generation

\n
    \n
  1. choose a random ephemeral key \\(k_{E} \\in {0,1,2,..,p-2}\\) such that \\(gcd(k_{E}, p-1) = 1\\)
    2. \n
  2. compute the signatue parameters:
    \\[r \\equiv \\alpha^{k_{E}} \\bmod p\\]
    \\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\]
    3. \n
\n
\n

on the receiving side, the signature is verified as \\(ver_{k_{pub}}(x,(r,s))\\) using the public key, the signature and the message.

\n
\n

Elgamal Signature Verification

\n
    \n
  1. comput the value
    \\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\]
    2. \n
  2. the verification follows form
    \\begin{equation}
    t =
    \\begin{cases}
    \\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr
    \\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    3. \n
\n
\n

Digital Signature Algorithm (DSA)

The native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks
that can threaten the Elgamal scheme are not applicable.

\n

key generation

\n

key Generation for DSA

\n
    \n
  1. Generate a prime \\(p\\) with \\(2^1023 < p < 2^1024\\)
    2. \n
  2. find a prime divisor \\(q\\) of \\(p-1\\) \\(2^159 < q < 2^160\\)
    3. \n
  3. Find an element \\(\\alpha\\) with \\( ord(\\alpha) = q\\), i.e., \\alpha genertes the subgroup with \\(q\\) elements.
    4. \n
  4. choose a random integer \\(d\\) with \\(0 < d < q\\).
    5. \n
  5. compute \\(\\beta \\equiv \\alpha^{d} \\bmod p\\).
    the keys are now:
    \\(k_{pub} = (p, q, \\alpha, \\beta)\\)
    \\(k_{pr}= (d)\\)
    6. \n
\n
\n

The central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\(Z_{p}*{\\ast}\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\(Z_{p}^{\\ast}\\). this set-up yields shorter signature.

\n

Signature and Verification

As in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\((r,s)\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit.

\n
\n

DSA signature generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\(0 < k_{E} < q\\).
    2. \n
  2. compute \\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\)
    3. \n
  3. compute \\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\)
    4. \n
\n
\n

The signature verification process is as follows:

\n
\n

DSA signature verification

\n
    \n
  1. compute auxilary value \\(w \\equiv s^{-1} \\bmod q\\).
    2. \n
  2. compute auxilary value \\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\).
    3. \n
  3. compute auxilary value \\(u_{2} \\equiv w \\cdot r \\bmod q\\).
    4. \n
  4. compute \\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\).
    5. \n
  5. the verification \\(ver_{k_{pub}}(x, (r,s))\\) folows from
    \\begin{equation}
    v =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

Elliptic Curve Digital Signature Algorithm (ECDSA)

Elliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures.
The ECDSA standard is defined for elliptic curves over prime fields \\(Z_{p}\\) adn Galois fields \\(GF(2^m)\\). the former is often preferred in practice, and we only introduce this one in what follows

\n

key generation

\n

Key Generation for ECDSA

\n
    \n
  1. Use and elliptic curve E with
    2. \n
\n
    \n
  • modulus p
    • \n
  • coefficients a and b
    • \n
  • a point A which generates a cyclic group of prime order q.
    • \n
\n
    \n
  1. choose a random integer d with \\(0 < d < q\\)
    2. \n
  2. compute \\(B = dA\\).
    The keys are now
    \\(k_{pub} = (p,a,b,q,A,B)\\)
    \\(k_{pr} = (d)\\)
    3. \n
\n
\n

Signature and Verification

\n

ECDSA Signature Generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\( 0 < k_{E} < q\\).
    2. \n
  2. compute \\(R = k_{E}A\\)
    3. \n
  3. Let \\(r = x_{R}\\)
    4. \n
  4. compute \\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\)
    5. \n
\n
\n

the signature verification process is as follows

\n
\n

ECDSA Signature Verification

\n
    \n
  1. Compute auxiliary value \\(w \\equiv s^{-1} \\bmod q\\)
    2. \n
  2. compute auxilary value \\(u_1 \\equiv w \\cdot h(x) \\bmod q\\)
    3. \n
  3. compute auxiliary value \\(u_2 = w \\cdot r \\bmod q\\)
    4. \n
  4. compute \\(P = u_1 A + u_2 B\\).
    5. \n
  5. the verification \\(ver{k_{pub}}(x, (r,s))\\) follows from
    \\begin{equation}
    x_{P} =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

The point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

Elgamal Digital Signature Scheme

The Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.

\n

key generation

\n
    \n
  1. Choose a large prime \\(p\\).
    2. \n
  2. Choose a primitive element \\(\\alpha\\) of \\(Z_{p}^{\\ast}\\), or a subgroup of \\(Z_{p}^{\\ast}\\).
    3. \n
  3. Choose a random integer \\(d \\in {2,3,…,p-2}\\)
    4. \n
  4. Compute \\(\\beta = \\alpha^{d} \\bmod p\\)
    5. \n
\n
\n

The public key is now formed by \\(k_{pub} = (p, \\alpha, \\beta)\\), and the private key by \\(k_{pr}=d\\)
\\(Z_{p}^{\\ast}\\) is the set of integers who are smaller than \\(p\\) and coprime to \\(p\\)

\n

signature and verification

Usign the private ey and parameters of the public key, the signature
\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\]
\\(x\\) is the message. \\(k_{E}\\) is a random value, which forms an ephemeral private key

\n
\n

Elgamal Signature Generation

\n
    \n
  1. choose a random ephemeral key \\(k_{E} \\in {0,1,2,..,p-2}\\) such that \\(gcd(k_{E}, p-1) = 1\\)
    2. \n
  2. compute the signatue parameters:
    \\[r \\equiv \\alpha^{k_{E}} \\bmod p\\]
    \\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\]
    3. \n
\n
\n

on the receiving side, the signature is verified as \\(ver_{k_{pub}}(x,(r,s))\\) using the public key, the signature and the message.

\n
\n

Elgamal Signature Verification

\n
    \n
  1. comput the value
    \\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\]
    2. \n
  2. the verification follows form
    \\begin{equation}
    t =
    \\begin{cases}
    \\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr
    \\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    3. \n
\n
\n

Digital Signature Algorithm (DSA)

The native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks
that can threaten the Elgamal scheme are not applicable.

\n

key generation

\n

key Generation for DSA

\n
    \n
  1. Generate a prime \\(p\\) with \\(2^1023 < p < 2^1024\\)
    2. \n
  2. find a prime divisor \\(q\\) of \\(p-1\\) \\(2^159 < q < 2^160\\)
    3. \n
  3. Find an element \\(\\alpha\\) with \\( ord(\\alpha) = q\\), i.e., \\alpha genertes the subgroup with \\(q\\) elements.
    4. \n
  4. choose a random integer \\(d\\) with \\(0 < d < q\\).
    5. \n
  5. compute \\(\\beta \\equiv \\alpha^{d} \\bmod p\\).
    the keys are now:
    \\(k_{pub} = (p, q, \\alpha, \\beta)\\)
    \\(k_{pr}= (d)\\)
    6. \n
\n
\n

The central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\(Z_{p}*{\\ast}\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\(Z_{p}^{\\ast}\\). this set-up yields shorter signature.

\n

Signature and Verification

As in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\((r,s)\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit.

\n
\n

DSA signature generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\(0 < k_{E} < q\\).
    2. \n
  2. compute \\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\)
    3. \n
  3. compute \\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\)
    4. \n
\n
\n

The signature verification process is as follows:

\n
\n

DSA signature verification

\n
    \n
  1. compute auxilary value \\(w \\equiv s^{-1} \\bmod q\\).
    2. \n
  2. compute auxilary value \\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\).
    3. \n
  3. compute auxilary value \\(u_{2} \\equiv w \\cdot r \\bmod q\\).
    4. \n
  4. compute \\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\).
    5. \n
  5. the verification \\(ver_{k_{pub}}(x, (r,s))\\) folows from
    \\begin{equation}
    v =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

Elliptic Curve Digital Signature Algorithm (ECDSA)

Elliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures.
The ECDSA standard is defined for elliptic curves over prime fields \\(Z_{p}\\) adn Galois fields \\(GF(2^m)\\). the former is often preferred in practice, and we only introduce this one in what follows

\n

key generation

\n

Key Generation for ECDSA

\n
    \n
  1. Use and elliptic curve E with
    2. \n
\n
    \n
  • modulus p
    • \n
  • coefficients a and b
    • \n
  • a point A which generates a cyclic group of prime order q.
    • \n
\n
    \n
  1. choose a random integer d with \\(0 < d < q\\)
    2. \n
  2. compute \\(B = dA\\).
    The keys are now
    \\(k_{pub} = (p,a,b,q,A,B)\\)
    \\(k_{pr} = (d)\\)
    3. \n
\n
\n

Signature and Verification

\n

ECDSA Signature Generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\( 0 < k_{E} < q\\).
    2. \n
  2. compute \\(R = k_{E}A\\)
    3. \n
  3. Let \\(r = x_{R}\\)
    4. \n
  4. compute \\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\)
    5. \n
\n
\n

the signature verification process is as follows

\n
\n

ECDSA Signature Verification

\n
    \n
  1. Compute auxiliary value \\(w \\equiv s^{-1} \\bmod q\\)
    2. \n
  2. compute auxilary value \\(u_1 \\equiv w \\cdot h(x) \\bmod q\\)
    3. \n
  3. compute auxiliary value \\(u_2 = w \\cdot r \\bmod q\\)
    4. \n
  4. compute \\(P = u_1 A + u_2 B\\).
    5. \n
  5. the verification \\(ver{k_{pub}}(x, (r,s))\\) follows from
    \\begin{equation}
    x_{P} =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

The point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.

\n"},{"title":"go reflect","date":"2023-03-02T02:44:50.000Z","_content":"\n## introduction\nReflection is the ability of a program to examine its own structure, particularly through types. \n\n## type and interfaces\nGo is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare\n```go\ntype MyInt int\n\nvar i int\nvar j MyInt\n```\n**The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.**\n\nOne important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:\n```go\ninterface{}\n```\nor its equivalent alias,\n```go\nany\n```\nIt represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.\na variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.\n\n## the representation of an interface\nA variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.\nFor instance, after\n```golang\nvar r io.Reader\ntty, err := os.OpenFile(\"/dev/tty\", os.O_RDWR, 0)\nif err != nil {\n return nil, err\n}\nr = tty\n```\nr contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:\n```go\nvar w io.Writer\nw = r.(io.Writer)\n```\nThe expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r. \nOne important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.\n\n## the first law of reflection\n1. Reflection goes from interface value to reflection object\nAt the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. `reflect.TypeOf` and `reflect.ValueOf`, retrieve `reflect.Type` and `reflect.Value` pieces out of an interface value.\n```go\nvar x float64 = 3.4\nfmt.Println(\"type:\", reflect.TypeOf(x))\n```\n```\ntype: float64\n```\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type())\nfmt.Println(\"kind is float64:\", v.Kind() == reflect.Float64)\nfmt.Println(\"value:\", v.Float())\n```\n```\ntype: float64\nkind is float64: true\nvalue: 3.4\n```\nThere are also methods like SetInt and SetFloat. **to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value**: int64 for all the signed integers, for instance. \n```go\nvar x uint8 = 'x'\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type()) // uint8.\nfmt.Println(\"kind is uint8: \", v.Kind() == reflect.Uint8) // true.\nx = uint8(v.Uint()) // v.Uint returns a uint64.\n```\n\n2. Reflection goes from reflection object to interface value.\nGiven a reflect.Value we can recover an interface value using the Interface method;\n```go\n// Interface returns v's current value as an interface{}.\n// It is equivalent to:\n//\n//\tvar i interface{} = (v's underlying value)\nfunc (v Value) Interface() interface{}\n```\n```go\ny := v.Interface().(float64) // y will have type float64.\nfmt.Println(y)\n```\n\n3. To modify a reflection object, the value must be settable.\nThe CanSet method of Value reports the settability of a Value; in our case,\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: false\n```\nwe pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement `v.SetFloat(7.1)` were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.\nLet’s do that. \n```go\nvar x float64 = 3.4\np := reflect.ValueOf(&x) // Note: take the address of x.\nfmt.Println(\"type of p:\", p.Type())\nfmt.Println(\"settability of p:\", p.CanSet())\n```\nThe reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:\n```go\nv := p.Elem()\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: true\n```\n\n## structs\n\n```go\ntype T struct {\n A int\n B string\n}\nt := T{23, \"skidoo\"}\ns := reflect.ValueOf(&t).Elem()\ntypeOfT := s.Type()\nfor i := 0; i < s.NumField(); i++ {\n f := s.Field(i)\n fmt.Printf(\"%d: %s %s = %v\\n\", i,\n typeOfT.Field(i).Name, f.Type(), f.Interface())\n}\n```\n```\n0: A int = 23\n1: B string = skidoo\n```\nThere’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.\nBecause s contains a settable reflection object, we can modify the fields of the structure.\n```\ns.Field(0).SetInt(77)\ns.Field(1).SetString(\"Sunset Strip\")\nfmt.Println(\"t is now\", t)\n```\nIf we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.\n\n## references\n- [official blog](https://go.dev/blog/laws-of-reflection)\n- [go data structure: interface](https://research.swtch.com/interfaces)","source":"_posts/golang/go-reflect.md","raw":"---\ntitle: go reflect\ndate: 2023-03-02 10:44:50\ntags: [golang]\n---\n\n## introduction\nReflection is the ability of a program to examine its own structure, particularly through types. \n\n## type and interfaces\nGo is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare\n```go\ntype MyInt int\n\nvar i int\nvar j MyInt\n```\n**The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.**\n\nOne important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:\n```go\ninterface{}\n```\nor its equivalent alias,\n```go\nany\n```\nIt represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.\na variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.\n\n## the representation of an interface\nA variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.\nFor instance, after\n```golang\nvar r io.Reader\ntty, err := os.OpenFile(\"/dev/tty\", os.O_RDWR, 0)\nif err != nil {\n return nil, err\n}\nr = tty\n```\nr contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:\n```go\nvar w io.Writer\nw = r.(io.Writer)\n```\nThe expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r. \nOne important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.\n\n## the first law of reflection\n1. Reflection goes from interface value to reflection object\nAt the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. `reflect.TypeOf` and `reflect.ValueOf`, retrieve `reflect.Type` and `reflect.Value` pieces out of an interface value.\n```go\nvar x float64 = 3.4\nfmt.Println(\"type:\", reflect.TypeOf(x))\n```\n```\ntype: float64\n```\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type())\nfmt.Println(\"kind is float64:\", v.Kind() == reflect.Float64)\nfmt.Println(\"value:\", v.Float())\n```\n```\ntype: float64\nkind is float64: true\nvalue: 3.4\n```\nThere are also methods like SetInt and SetFloat. **to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value**: int64 for all the signed integers, for instance. \n```go\nvar x uint8 = 'x'\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type()) // uint8.\nfmt.Println(\"kind is uint8: \", v.Kind() == reflect.Uint8) // true.\nx = uint8(v.Uint()) // v.Uint returns a uint64.\n```\n\n2. Reflection goes from reflection object to interface value.\nGiven a reflect.Value we can recover an interface value using the Interface method;\n```go\n// Interface returns v's current value as an interface{}.\n// It is equivalent to:\n//\n//\tvar i interface{} = (v's underlying value)\nfunc (v Value) Interface() interface{}\n```\n```go\ny := v.Interface().(float64) // y will have type float64.\nfmt.Println(y)\n```\n\n3. To modify a reflection object, the value must be settable.\nThe CanSet method of Value reports the settability of a Value; in our case,\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: false\n```\nwe pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement `v.SetFloat(7.1)` were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.\nLet’s do that. \n```go\nvar x float64 = 3.4\np := reflect.ValueOf(&x) // Note: take the address of x.\nfmt.Println(\"type of p:\", p.Type())\nfmt.Println(\"settability of p:\", p.CanSet())\n```\nThe reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:\n```go\nv := p.Elem()\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: true\n```\n\n## structs\n\n```go\ntype T struct {\n A int\n B string\n}\nt := T{23, \"skidoo\"}\ns := reflect.ValueOf(&t).Elem()\ntypeOfT := s.Type()\nfor i := 0; i < s.NumField(); i++ {\n f := s.Field(i)\n fmt.Printf(\"%d: %s %s = %v\\n\", i,\n typeOfT.Field(i).Name, f.Type(), f.Interface())\n}\n```\n```\n0: A int = 23\n1: B string = skidoo\n```\nThere’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.\nBecause s contains a settable reflection object, we can modify the fields of the structure.\n```\ns.Field(0).SetInt(77)\ns.Field(1).SetString(\"Sunset Strip\")\nfmt.Println(\"t is now\", t)\n```\nIf we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.\n\n## references\n- [official blog](https://go.dev/blog/laws-of-reflection)\n- [go data structure: interface](https://research.swtch.com/interfaces)","slug":"golang/go-reflect","published":1,"updated":"2023-06-18T06:42:44.968Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzm000cofsj603lgdqw","content":"

introduction

Reflection is the ability of a program to examine its own structure, particularly through types.

\n

type and interfaces

Go is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare

\n
1
2
3
4
type MyInt int

var i int
var j MyInt
\n

The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.

\n

One important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:

\n
1
interface{}
\n

or its equivalent alias,

\n
1
any
\n

It represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.
a variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.

\n

the representation of an interface

A variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.
For instance, after

\n
1
2
3
4
5
6
var r io.Reader
tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
if err != nil {
    return nil, err
}
r = tty
\n

r contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:

\n
1
2
var w io.Writer
w = r.(io.Writer)
\n

The expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r.
One important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.

\n

the first law of reflection

    \n

  1. Reflection goes from interface value to reflection object
    At the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. reflect.TypeOf and reflect.ValueOf, retrieve reflect.Type and reflect.Value pieces out of an interface value.

    \n
    1
    2
    var x float64 = 3.4
    fmt.Println("type:", reflect.TypeOf(x))
    \n
    1
    type: float64
    \n
    1
    2
    3
    4
    5
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())
    fmt.Println("kind is float64:", v.Kind() == reflect.Float64)
    fmt.Println("value:", v.Float())
    \n
    1
    2
    3
    type: float64
    kind is float64: true
    value: 3.4
    \n

    There are also methods like SetInt and SetFloat. to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value: int64 for all the signed integers, for instance.

    \n
    1
    2
    3
    4
    5
    var x uint8 = 'x'
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())                            // uint8.
    fmt.Println("kind is uint8: ", v.Kind() == reflect.Uint8) // true.
    x = uint8(v.Uint())                                       // v.Uint returns a uint64.
    \n
    2. \n

  2. Reflection goes from reflection object to interface value.
    Given a reflect.Value we can recover an interface value using the Interface method;

    \n
    1
    2
    3
    4
    5
    // Interface returns v's current value as an interface{}.
    // It is equivalent to:
    //
    //\tvar i interface{} = (v's underlying value)
    func (v Value) Interface() interface{}
    \n
    1
    2
    y := v.Interface().(float64) // y will have type float64.
    fmt.Println(y)
    \n
    3. \n

  3. To modify a reflection object, the value must be settable.
    The CanSet method of Value reports the settability of a Value; in our case,

    \n
    1
    2
    3
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: false
    \n

    we pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement v.SetFloat(7.1) were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.
    Let’s do that.

    \n
    1
    2
    3
    4
    var x float64 = 3.4
    p := reflect.ValueOf(&x) // Note: take the address of x.
    fmt.Println("type of p:", p.Type())
    fmt.Println("settability of p:", p.CanSet())
    \n

    The reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:

    \n
    1
    2
    v := p.Elem()
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: true
    4. \n
\n

structs

1
2
3
4
5
6
7
8
9
10
11
12
type T struct {
    A int
    B string
}
t := T{23, "skidoo"}
s := reflect.ValueOf(&t).Elem()
typeOfT := s.Type()
for i := 0; i < s.NumField(); i++ {
    f := s.Field(i)
    fmt.Printf("%d: %s %s = %v\\n", i,
        typeOfT.Field(i).Name, f.Type(), f.Interface())
}
\n
1
2
0: A int = 23
1: B string = skidoo
\n

There’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.
Because s contains a settable reflection object, we can modify the fields of the structure.

\n
1
2
3
s.Field(0).SetInt(77)
s.Field(1).SetString("Sunset Strip")
fmt.Println("t is now", t)
\n

If we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

Reflection is the ability of a program to examine its own structure, particularly through types.

\n

type and interfaces

Go is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare

\n
1
2
3
4
type MyInt int

var i int
var j MyInt
\n

The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.

\n

One important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:

\n
1
interface{}
\n

or its equivalent alias,

\n
1
any
\n

It represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.
a variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.

\n

the representation of an interface

A variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.
For instance, after

\n
1
2
3
4
5
6
var r io.Reader
tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
if err != nil {
    return nil, err
}
r = tty
\n

r contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:

\n
1
2
var w io.Writer
w = r.(io.Writer)
\n

The expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r.
One important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.

\n

the first law of reflection

    \n

  1. Reflection goes from interface value to reflection object
    At the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. reflect.TypeOf and reflect.ValueOf, retrieve reflect.Type and reflect.Value pieces out of an interface value.

    \n
    1
    2
    var x float64 = 3.4
    fmt.Println("type:", reflect.TypeOf(x))
    \n
    1
    type: float64
    \n
    1
    2
    3
    4
    5
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())
    fmt.Println("kind is float64:", v.Kind() == reflect.Float64)
    fmt.Println("value:", v.Float())
    \n
    1
    2
    3
    type: float64
    kind is float64: true
    value: 3.4
    \n

    There are also methods like SetInt and SetFloat. to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value: int64 for all the signed integers, for instance.

    \n
    1
    2
    3
    4
    5
    var x uint8 = 'x'
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())                            // uint8.
    fmt.Println("kind is uint8: ", v.Kind() == reflect.Uint8) // true.
    x = uint8(v.Uint())                                       // v.Uint returns a uint64.
    \n
    2. \n

  2. Reflection goes from reflection object to interface value.
    Given a reflect.Value we can recover an interface value using the Interface method;

    \n
    1
    2
    3
    4
    5
    // Interface returns v's current value as an interface{}.
    // It is equivalent to:
    //
    //\tvar i interface{} = (v's underlying value)
    func (v Value) Interface() interface{}
    \n
    1
    2
    y := v.Interface().(float64) // y will have type float64.
    fmt.Println(y)
    \n
    3. \n

  3. To modify a reflection object, the value must be settable.
    The CanSet method of Value reports the settability of a Value; in our case,

    \n
    1
    2
    3
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: false
    \n

    we pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement v.SetFloat(7.1) were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.
    Let’s do that.

    \n
    1
    2
    3
    4
    var x float64 = 3.4
    p := reflect.ValueOf(&x) // Note: take the address of x.
    fmt.Println("type of p:", p.Type())
    fmt.Println("settability of p:", p.CanSet())
    \n

    The reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:

    \n
    1
    2
    v := p.Elem()
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: true
    4. \n
\n

structs

1
2
3
4
5
6
7
8
9
10
11
12
type T struct {
    A int
    B string
}
t := T{23, "skidoo"}
s := reflect.ValueOf(&t).Elem()
typeOfT := s.Type()
for i := 0; i < s.NumField(); i++ {
    f := s.Field(i)
    fmt.Printf("%d: %s %s = %v\\n", i,
        typeOfT.Field(i).Name, f.Type(), f.Interface())
}
\n
1
2
0: A int = 23
1: B string = skidoo
\n

There’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.
Because s contains a settable reflection object, we can modify the fields of the structure.

\n
1
2
3
s.Field(0).SetInt(77)
s.Field(1).SetString("Sunset Strip")
fmt.Println("t is now", t)
\n

If we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.

\n

references

\n"},{"title":"paillier encryption","date":"2023-02-23T13:25:41.000Z","_content":"\n\n\n\n## fundamentals\n1. fundamental theorem of arighmetic\nthe fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors [wiki](https://en.wikipedia.org/wiki/Fundamental_theorem_of_arithmetic)\n2. Euler's totient function\nIn number theory, Euler's totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\\\( \\phi (n) \\\\), and may also be called Euler's phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\\\( Z_{n}^{\\ast } \\\\), and \\\\[ \\phi(n) = |Z_n^{\\ast }| \\\\]\n3. if p is prime, then \\\\( Z_p^{\\ast } = Z_p \\\\), \\\\( \\phi(p) = p-1 \\\\)\n4. if p is prime, for any integer r, then \\\\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\\\)\n5. Euler's totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\\\(\\phi(mn) = \\phi(m)\\phi(n)\\\\)\n6. Euler's product formula, it states\n\\\\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\\\]\nwhere the product is over the distinct prime numbers dividing n.\n7. Euler's theorem\nif \\\\(a\\\\) and \\\\(n\\\\) are coprime positive integers, and \\\\( \\phi(n)\\\\) is Euler's totient function, then \\\\(a\\\\) raised to the power \\\\(\\phi(n)\\\\) is congruent to 1 modulo n; that is\n\\\\[a^{\\phi(n)} \\equiv 1 \\bmod n\\\\]\n8. according to 7, we have \\\\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\\\). then\n\\\\[ a^{-1} = a^{\\phi(n)-1} \\\\]\n9. Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\n10. Binomial theorem\nit states\n\\\\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + ...\\\\]\nobserve that, the higher degree could be divided by \\\\(n^2\\\\). we have\n\\\\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\\\]\ntherefore, \\\\( y - 1 \\equiv nx \\bmod n^2 \\\\). then we have\n\\\\[ x \\equiv \\frac{y-1}{n} \\bmod n \\\\].\nIn paillier, later we define \\\\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\\\)\ntherefore\n\\\\[ L(y \\bmod n^2) \\equiv x \\bmod n \\\\]\n\n\n## Paillier\n1. key generation\n`KeyGen() -> (pk, sk)`\nrandomly select two big prime numbers \\\\(p, q\\\\). it shoud satisfy \\\\(gcd(pq, (p-1)(q-1)) =1 \\\\), \\\\(p\\\\) and \\\\(q\\\\) should have similar bit length. let \\\\( n = pq \\\\), \\\\(\\lambda = lcm(p-1, q-1)\\\\). randomly sample \\\\( g \\in Z_{n^2}^{\\ast}\\\\). to simplify, let \\\\( g = n+1\\\\). we have\n\\\\[ pk=(n,g) \\\\]\n\\\\[ sk = (\\lambda)\\\\]\n\n2. encryption\n`Enc(pk, m) -> c`\nrandomly sample \\\\( r \\in Z_{n}^{\\ast}\\\\), then also have \\\\( r \\in Z_{n^2}^{\\ast}\\\\), cypher is calculated\n\\\\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\\\]\n\n3. Decryption\n`Dec(sk, c) -> m`\nLet \\\\(L(x) = \\frac{x-1}{n} \\\\), we have message\n\\\\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\\\]\n\n4. proof of correctness\nbased on Eq(1), we have \\\\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\\\]\nwhere \\\\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\\\), which is proved by Carmichael theorem later on. then Eq(3) becomes\n \\\\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nsince \\\\( g = n+1\\\\), we have\n\\\\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nAccording to Eq(0.2), we have\n\\\\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\\\]\n\\\\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\\\]\ntherefore, based on definition given by Eq(0.3) we have\n\\\\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\\\]\nSubstitute Eq(1.6) into Eq(1.8), we have\n\\\\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\\\]\nFurther, we have\n\\\\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\\\]\nSub Eq(1.7) into Eq(1.10), we have\n\\\\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\\\]\nAt last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)\n\\\\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\\\]\nproved!!!\n\n5. Carmichael theorem\nIn number theory, a branch of mathematics, the Carmichael function \\\\(λ(n)\\\\) of a positive integer n is the smallest positive integer m such that \\\\(a^{m}\\equiv 1{\\pmod {n}}\\\\) (similar but different from Euler's totient function). Carmichael's λ function, the reduced totient function, and the least universal exponent function\n![carmichael theorem](/images/paillier/carmichael_thorem.png)\n![](/images/paillier/carmichael_thorem_2.png)\nlet \\\\( n = pq\\\\), where p and q are prime numbers; \\\\( \\phi(n)\\\\) is the Euler's totient function. Let \\\\(\\lambda(n)\\\\) denotes carmichael function. We have \\\\(\\phi(n)=(p-1)(q-1)\\\\) and \\\\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\\\).\n\nSince \\\\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\\\) (according to Eq(0.1)). Thereby, for any \\\\( w \\in Z_{n^2}^{\\ast}\\\\)\n\\\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\\\]\n\n\\\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\\\]\nEq(1.13) is just Carmichael's function\n\nBased on Carmichael's theorem\n\\\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\\\] \ntherefore, we have\n\n\\\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\\\]\n\n6. Addition homomorphic\n![homomorphic addition](/images/paillier/homomorphic_addition.png)\n\n7. Multiplication homomorphic \n![homomorphic multiplication](/images/paillier/homomorphic_mul.png)\n## references\n- [csdn post](https://blog.csdn.net/qq_42328228/article/details/109349590)","source":"_posts/cryptography/paillier-encryption.md","raw":"---\ntitle: paillier encryption\ndate: 2023-02-23 21:25:41\ntags: [cryptography]\n---\n\n\n\n\n## fundamentals\n1. fundamental theorem of arighmetic\nthe fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors [wiki](https://en.wikipedia.org/wiki/Fundamental_theorem_of_arithmetic)\n2. Euler's totient function\nIn number theory, Euler's totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\\\( \\phi (n) \\\\), and may also be called Euler's phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\\\( Z_{n}^{\\ast } \\\\), and \\\\[ \\phi(n) = |Z_n^{\\ast }| \\\\]\n3. if p is prime, then \\\\( Z_p^{\\ast } = Z_p \\\\), \\\\( \\phi(p) = p-1 \\\\)\n4. if p is prime, for any integer r, then \\\\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\\\)\n5. Euler's totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\\\(\\phi(mn) = \\phi(m)\\phi(n)\\\\)\n6. Euler's product formula, it states\n\\\\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\\\]\nwhere the product is over the distinct prime numbers dividing n.\n7. Euler's theorem\nif \\\\(a\\\\) and \\\\(n\\\\) are coprime positive integers, and \\\\( \\phi(n)\\\\) is Euler's totient function, then \\\\(a\\\\) raised to the power \\\\(\\phi(n)\\\\) is congruent to 1 modulo n; that is\n\\\\[a^{\\phi(n)} \\equiv 1 \\bmod n\\\\]\n8. according to 7, we have \\\\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\\\). then\n\\\\[ a^{-1} = a^{\\phi(n)-1} \\\\]\n9. Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\n10. Binomial theorem\nit states\n\\\\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + ...\\\\]\nobserve that, the higher degree could be divided by \\\\(n^2\\\\). we have\n\\\\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\\\]\ntherefore, \\\\( y - 1 \\equiv nx \\bmod n^2 \\\\). then we have\n\\\\[ x \\equiv \\frac{y-1}{n} \\bmod n \\\\].\nIn paillier, later we define \\\\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\\\)\ntherefore\n\\\\[ L(y \\bmod n^2) \\equiv x \\bmod n \\\\]\n\n\n## Paillier\n1. key generation\n`KeyGen() -> (pk, sk)`\nrandomly select two big prime numbers \\\\(p, q\\\\). it shoud satisfy \\\\(gcd(pq, (p-1)(q-1)) =1 \\\\), \\\\(p\\\\) and \\\\(q\\\\) should have similar bit length. let \\\\( n = pq \\\\), \\\\(\\lambda = lcm(p-1, q-1)\\\\). randomly sample \\\\( g \\in Z_{n^2}^{\\ast}\\\\). to simplify, let \\\\( g = n+1\\\\). we have\n\\\\[ pk=(n,g) \\\\]\n\\\\[ sk = (\\lambda)\\\\]\n\n2. encryption\n`Enc(pk, m) -> c`\nrandomly sample \\\\( r \\in Z_{n}^{\\ast}\\\\), then also have \\\\( r \\in Z_{n^2}^{\\ast}\\\\), cypher is calculated\n\\\\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\\\]\n\n3. Decryption\n`Dec(sk, c) -> m`\nLet \\\\(L(x) = \\frac{x-1}{n} \\\\), we have message\n\\\\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\\\]\n\n4. proof of correctness\nbased on Eq(1), we have \\\\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\\\]\nwhere \\\\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\\\), which is proved by Carmichael theorem later on. then Eq(3) becomes\n \\\\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nsince \\\\( g = n+1\\\\), we have\n\\\\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nAccording to Eq(0.2), we have\n\\\\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\\\]\n\\\\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\\\]\ntherefore, based on definition given by Eq(0.3) we have\n\\\\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\\\]\nSubstitute Eq(1.6) into Eq(1.8), we have\n\\\\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\\\]\nFurther, we have\n\\\\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\\\]\nSub Eq(1.7) into Eq(1.10), we have\n\\\\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\\\]\nAt last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)\n\\\\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\\\]\nproved!!!\n\n5. Carmichael theorem\nIn number theory, a branch of mathematics, the Carmichael function \\\\(λ(n)\\\\) of a positive integer n is the smallest positive integer m such that \\\\(a^{m}\\equiv 1{\\pmod {n}}\\\\) (similar but different from Euler's totient function). Carmichael's λ function, the reduced totient function, and the least universal exponent function\n![carmichael theorem](/images/paillier/carmichael_thorem.png)\n![](/images/paillier/carmichael_thorem_2.png)\nlet \\\\( n = pq\\\\), where p and q are prime numbers; \\\\( \\phi(n)\\\\) is the Euler's totient function. Let \\\\(\\lambda(n)\\\\) denotes carmichael function. We have \\\\(\\phi(n)=(p-1)(q-1)\\\\) and \\\\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\\\).\n\nSince \\\\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\\\) (according to Eq(0.1)). Thereby, for any \\\\( w \\in Z_{n^2}^{\\ast}\\\\)\n\\\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\\\]\n\n\\\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\\\]\nEq(1.13) is just Carmichael's function\n\nBased on Carmichael's theorem\n\\\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\\\] \ntherefore, we have\n\n\\\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\\\]\n\n6. Addition homomorphic\n![homomorphic addition](/images/paillier/homomorphic_addition.png)\n\n7. Multiplication homomorphic \n![homomorphic multiplication](/images/paillier/homomorphic_mul.png)\n## references\n- [csdn post](https://blog.csdn.net/qq_42328228/article/details/109349590)","slug":"cryptography/paillier-encryption","published":1,"updated":"2023-04-24T06:31:44.177Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzn000fofsj4keu9ail","content":"\n\n\n

fundamentals

    \n
  1. fundamental theorem of arighmetic
    the fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors wiki
    2. \n
  2. Euler’s totient function
    In number theory, Euler’s totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\( \\phi (n) \\), and may also be called Euler’s phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\( Z_{n}^{\\ast } \\), and \\[ \\phi(n) = |Z_n^{\\ast }| \\]
    3. \n
  3. if p is prime, then \\( Z_p^{\\ast } = Z_p \\), \\( \\phi(p) = p-1 \\)
    4. \n
  4. if p is prime, for any integer r, then \\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\)
    5. \n
  5. Euler’s totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\(\\phi(mn) = \\phi(m)\\phi(n)\\)
    6. \n
  6. Euler’s product formula, it states
    \\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\]
    where the product is over the distinct prime numbers dividing n.
    7. \n
  7. Euler’s theorem
    if \\(a\\) and \\(n\\) are coprime positive integers, and \\( \\phi(n)\\) is Euler’s totient function, then \\(a\\) raised to the power \\(\\phi(n)\\) is congruent to 1 modulo n; that is
    \\[a^{\\phi(n)} \\equiv 1 \\bmod n\\]
    8. \n
  8. according to 7, we have \\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\). then
    \\[ a^{-1} = a^{\\phi(n)-1} \\]
    9. \n
  9. Fermat’s little theorem
    Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
    \\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
    \\[ a^{p} \\equiv a \\bmod p\\]
    10. \n
  10. Binomial theorem
    it states
    \\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + …\\]
    observe that, the higher degree could be divided by \\(n^2\\). we have
    \\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\]
    therefore, \\( y - 1 \\equiv nx \\bmod n^2 \\). then we have
    \\[ x \\equiv \\frac{y-1}{n} \\bmod n \\].
    In paillier, later we define \\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\)
    therefore
    \\[ L(y \\bmod n^2) \\equiv x \\bmod n \\]
    11. \n
\n

Paillier

    \n

  1. key generation
    KeyGen() -> (pk, sk)
    randomly select two big prime numbers \\(p, q\\). it shoud satisfy \\(gcd(pq, (p-1)(q-1)) =1 \\), \\(p\\) and \\(q\\) should have similar bit length. let \\( n = pq \\), \\(\\lambda = lcm(p-1, q-1)\\). randomly sample \\( g \\in Z_{n^2}^{\\ast}\\). to simplify, let \\( g = n+1\\). we have
    \\[ pk=(n,g) \\]
    \\[ sk = (\\lambda)\\]

    \n
    2. \n

  2. encryption
    Enc(pk, m) -> c
    randomly sample \\( r \\in Z_{n}^{\\ast}\\), then also have \\( r \\in Z_{n^2}^{\\ast}\\), cypher is calculated
    \\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\]

    \n
    3. \n

  3. Decryption
    Dec(sk, c) -> m
    Let \\(L(x) = \\frac{x-1}{n} \\), we have message
    \\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\]

    \n
    4. \n

  4. proof of correctness
    based on Eq(1), we have \\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\]
    where \\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\), which is proved by Carmichael theorem later on. then Eq(3) becomes
    \\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\]
    since \\( g = n+1\\), we have
    \\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\]
    According to Eq(0.2), we have
    \\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\]
    \\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\]
    therefore, based on definition given by Eq(0.3) we have
    \\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\]
    Substitute Eq(1.6) into Eq(1.8), we have
    \\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\]
    Further, we have
    \\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\]
    Sub Eq(1.7) into Eq(1.10), we have
    \\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\]
    At last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)
    \\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\]
    proved!!!

    \n
    5. \n

  5. Carmichael theorem
    In number theory, a branch of mathematics, the Carmichael function \\(λ(n)\\) of a positive integer n is the smallest positive integer m such that \\(a^{m}\\equiv 1{\\pmod {n}}\\) (similar but different from Euler’s totient function). Carmichael’s λ function, the reduced totient function, and the least universal exponent function
    \"carmichael

    let \\( n = pq\\), where p and q are prime numbers; \\( \\phi(n)\\) is the Euler’s totient function. Let \\(\\lambda(n)\\) denotes carmichael function. We have \\(\\phi(n)=(p-1)(q-1)\\) and \\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\).

    \n
    6. \n
\n

Since \\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\) (according to Eq(0.1)). Thereby, for any \\( w \\in Z_{n^2}^{\\ast}\\)
\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\]

\n

\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\]
Eq(1.13) is just Carmichael’s function

\n

Based on Carmichael’s theorem
\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\]
therefore, we have

\n

\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\]

\n
    \n

  1. Addition homomorphic
    \"homomorphic

    \n
    2. \n

  2. Multiplication homomorphic
    \"homomorphic

    \n
    3. \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

fundamentals

    \n
  1. fundamental theorem of arighmetic
    the fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors wiki
    2. \n
  2. Euler’s totient function
    In number theory, Euler’s totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\( \\phi (n) \\), and may also be called Euler’s phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\( Z_{n}^{\\ast } \\), and \\[ \\phi(n) = |Z_n^{\\ast }| \\]
    3. \n
  3. if p is prime, then \\( Z_p^{\\ast } = Z_p \\), \\( \\phi(p) = p-1 \\)
    4. \n
  4. if p is prime, for any integer r, then \\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\)
    5. \n
  5. Euler’s totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\(\\phi(mn) = \\phi(m)\\phi(n)\\)
    6. \n
  6. Euler’s product formula, it states
    \\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\]
    where the product is over the distinct prime numbers dividing n.
    7. \n
  7. Euler’s theorem
    if \\(a\\) and \\(n\\) are coprime positive integers, and \\( \\phi(n)\\) is Euler’s totient function, then \\(a\\) raised to the power \\(\\phi(n)\\) is congruent to 1 modulo n; that is
    \\[a^{\\phi(n)} \\equiv 1 \\bmod n\\]
    8. \n
  8. according to 7, we have \\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\). then
    \\[ a^{-1} = a^{\\phi(n)-1} \\]
    9. \n
  9. Fermat’s little theorem
    Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
    \\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
    \\[ a^{p} \\equiv a \\bmod p\\]
    10. \n
  10. Binomial theorem
    it states
    \\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + …\\]
    observe that, the higher degree could be divided by \\(n^2\\). we have
    \\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\]
    therefore, \\( y - 1 \\equiv nx \\bmod n^2 \\). then we have
    \\[ x \\equiv \\frac{y-1}{n} \\bmod n \\].
    In paillier, later we define \\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\)
    therefore
    \\[ L(y \\bmod n^2) \\equiv x \\bmod n \\]
    11. \n
\n

Paillier

    \n

  1. key generation
    KeyGen() -> (pk, sk)
    randomly select two big prime numbers \\(p, q\\). it shoud satisfy \\(gcd(pq, (p-1)(q-1)) =1 \\), \\(p\\) and \\(q\\) should have similar bit length. let \\( n = pq \\), \\(\\lambda = lcm(p-1, q-1)\\). randomly sample \\( g \\in Z_{n^2}^{\\ast}\\). to simplify, let \\( g = n+1\\). we have
    \\[ pk=(n,g) \\]
    \\[ sk = (\\lambda)\\]

    \n
    2. \n

  2. encryption
    Enc(pk, m) -> c
    randomly sample \\( r \\in Z_{n}^{\\ast}\\), then also have \\( r \\in Z_{n^2}^{\\ast}\\), cypher is calculated
    \\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\]

    \n
    3. \n

  3. Decryption
    Dec(sk, c) -> m
    Let \\(L(x) = \\frac{x-1}{n} \\), we have message
    \\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\]

    \n
    4. \n

  4. proof of correctness
    based on Eq(1), we have \\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\]
    where \\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\), which is proved by Carmichael theorem later on. then Eq(3) becomes
    \\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\]
    since \\( g = n+1\\), we have
    \\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\]
    According to Eq(0.2), we have
    \\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\]
    \\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\]
    therefore, based on definition given by Eq(0.3) we have
    \\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\]
    Substitute Eq(1.6) into Eq(1.8), we have
    \\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\]
    Further, we have
    \\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\]
    Sub Eq(1.7) into Eq(1.10), we have
    \\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\]
    At last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)
    \\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\]
    proved!!!

    \n
    5. \n

  5. Carmichael theorem
    In number theory, a branch of mathematics, the Carmichael function \\(λ(n)\\) of a positive integer n is the smallest positive integer m such that \\(a^{m}\\equiv 1{\\pmod {n}}\\) (similar but different from Euler’s totient function). Carmichael’s λ function, the reduced totient function, and the least universal exponent function
    \"carmichael

    let \\( n = pq\\), where p and q are prime numbers; \\( \\phi(n)\\) is the Euler’s totient function. Let \\(\\lambda(n)\\) denotes carmichael function. We have \\(\\phi(n)=(p-1)(q-1)\\) and \\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\).

    \n
    6. \n
\n

Since \\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\) (according to Eq(0.1)). Thereby, for any \\( w \\in Z_{n^2}^{\\ast}\\)
\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\]

\n

\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\]
Eq(1.13) is just Carmichael’s function

\n

Based on Carmichael’s theorem
\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\]
therefore, we have

\n

\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\]

\n
    \n

  1. Addition homomorphic
    \"homomorphic

    \n
    2. \n

  2. Multiplication homomorphic
    \"homomorphic

    \n
    3. \n
\n

references

\n"},{"title":"two party ecdsa","date":"2023-02-07T06:29:26.000Z","_content":"\n\n\n## overview\nthis post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo's library (rust).\n\nUnlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\\\( k \\\\).\n\nIn this paper, we consider the specific case of two parties (and thus no honest majority) and con-struct a protocol that is approximately two orders of magnitude faster than the previous best.\n\n## Comparing ECDSA signing to EC-Schnorr signing\nIn both cases, the public verification key is an elliptic curve point \\\\( Q \\\\) and the private signing key is \\\\( x \\\\) such that \\\\( Q = x \\cdot G \\\\), where \\\\( G \\\\) is the generator point of an EC group of order \\\\( q \\\\).\n![schnorr ecdsa comparison](/images/two_party_ecdsa/schnorr_ecdsa_comparison.png)\n\nObserve that Schnorr signing can be easily distributed since the private key \\\\( x \\\\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\\\( s \\\\) includes \\\\( k^{-1} \\\\). Now, given shares \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 + k_2 = k \\bmod q\\\\) .It is very difficult to compute \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) such that \\\\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\\\)\n\n\ntwo-party protocols for ECDSA signing use multiplicative sharing of \\\\( x \\\\) and of \\\\( k \\\\). That is, the parties hold \\\\(x_1\\\\), \\\\(x_2\\\\) such that \\\\(x_1 \\cdot x_2 = x \\bmod q\\\\), and in each signing operation they generate \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 \\cdot k_2 = k \\bmod q\\\\). This enables them to easily compute \\\\(k^{-1}\\\\) since each party can locally compute \\\\(k_i^{\\prime} = k_i^{-1} \\bmod q\\\\), and then \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) are multiplicative shares of \\\\(k^{-1}\\\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\\\(P_1\\\\) can compute \\\\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\\\) and \\\\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\\\( P_2 \\\\) can compute \\\\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\\\), which will be an encryption of \n\n![paillier encryption](/images/two_party_ecdsa/paillier_enc.png)\n\nHowever, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\\\(k_1^{-1}\\\\) when the second party only has \\\\(R_1 = k_1 \\cdot G\\\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.\n\n## their results\n[WIP]\n\n## references\n- [original papger](https://eprint.iacr.org/2017/552.pdf)","source":"_posts/cryptography/two-party-ecdsa.md","raw":"---\ntitle: two party ecdsa\ndate: 2023-02-07 14:29:26\ntags: [cryptography,mpc,ecdsa]\n---\n\n\n\n## overview\nthis post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo's library (rust).\n\nUnlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\\\( k \\\\).\n\nIn this paper, we consider the specific case of two parties (and thus no honest majority) and con-struct a protocol that is approximately two orders of magnitude faster than the previous best.\n\n## Comparing ECDSA signing to EC-Schnorr signing\nIn both cases, the public verification key is an elliptic curve point \\\\( Q \\\\) and the private signing key is \\\\( x \\\\) such that \\\\( Q = x \\cdot G \\\\), where \\\\( G \\\\) is the generator point of an EC group of order \\\\( q \\\\).\n![schnorr ecdsa comparison](/images/two_party_ecdsa/schnorr_ecdsa_comparison.png)\n\nObserve that Schnorr signing can be easily distributed since the private key \\\\( x \\\\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\\\( s \\\\) includes \\\\( k^{-1} \\\\). Now, given shares \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 + k_2 = k \\bmod q\\\\) .It is very difficult to compute \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) such that \\\\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\\\)\n\n\ntwo-party protocols for ECDSA signing use multiplicative sharing of \\\\( x \\\\) and of \\\\( k \\\\). That is, the parties hold \\\\(x_1\\\\), \\\\(x_2\\\\) such that \\\\(x_1 \\cdot x_2 = x \\bmod q\\\\), and in each signing operation they generate \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 \\cdot k_2 = k \\bmod q\\\\). This enables them to easily compute \\\\(k^{-1}\\\\) since each party can locally compute \\\\(k_i^{\\prime} = k_i^{-1} \\bmod q\\\\), and then \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) are multiplicative shares of \\\\(k^{-1}\\\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\\\(P_1\\\\) can compute \\\\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\\\) and \\\\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\\\( P_2 \\\\) can compute \\\\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\\\), which will be an encryption of \n\n![paillier encryption](/images/two_party_ecdsa/paillier_enc.png)\n\nHowever, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\\\(k_1^{-1}\\\\) when the second party only has \\\\(R_1 = k_1 \\cdot G\\\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.\n\n## their results\n[WIP]\n\n## references\n- [original papger](https://eprint.iacr.org/2017/552.pdf)","slug":"cryptography/two-party-ecdsa","published":1,"updated":"2023-04-24T06:31:53.595Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzn000hofsj1c627es0","content":"\n\n\n

overview

this post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo’s library (rust).

\n

Unlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\( k \\).

\n

In this paper, we consider the specific case of two parties (and thus no honest majority) and con-struct a protocol that is approximately two orders of magnitude faster than the previous best.

\n

Comparing ECDSA signing to EC-Schnorr signing

In both cases, the public verification key is an elliptic curve point \\( Q \\) and the private signing key is \\( x \\) such that \\( Q = x \\cdot G \\), where \\( G \\) is the generator point of an EC group of order \\( q \\).
\"schnorr

\n

Observe that Schnorr signing can be easily distributed since the private key \\( x \\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\( s \\) includes \\( k^{-1} \\). Now, given shares \\(k_1\\), \\(k_2\\) such that \\(k_1 + k_2 = k \\bmod q\\) .It is very difficult to compute \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) such that \\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\)

\n

two-party protocols for ECDSA signing use multiplicative sharing of \\( x \\) and of \\( k \\). That is, the parties hold \\(x_1\\), \\(x_2\\) such that \\(x_1 \\cdot x_2 = x \\bmod q\\), and in each signing operation they generate \\(k_1\\), \\(k_2\\) such that \\(k_1 \\cdot k_2 = k \\bmod q\\). This enables them to easily compute \\(k^{-1}\\) since each party can locally compute \\(k_i^{\\prime} = k_i^{-1} \\bmod q\\), and then \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) are multiplicative shares of \\(k^{-1}\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\(P_1\\) can compute \\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\) and \\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\( P_2 \\) can compute \\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\), which will be an encryption of

\n

\"paillier

\n

However, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\(k_1^{-1}\\) when the second party only has \\(R_1 = k_1 \\cdot G\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.

\n

their results

[WIP]

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

overview

this post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo’s library (rust).

\n

Unlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\( k \\).

\n

In this paper, we consider the specific case of two parties (and thus no honest majority) and con-struct a protocol that is approximately two orders of magnitude faster than the previous best.

\n

Comparing ECDSA signing to EC-Schnorr signing

In both cases, the public verification key is an elliptic curve point \\( Q \\) and the private signing key is \\( x \\) such that \\( Q = x \\cdot G \\), where \\( G \\) is the generator point of an EC group of order \\( q \\).
\"schnorr

\n

Observe that Schnorr signing can be easily distributed since the private key \\( x \\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\( s \\) includes \\( k^{-1} \\). Now, given shares \\(k_1\\), \\(k_2\\) such that \\(k_1 + k_2 = k \\bmod q\\) .It is very difficult to compute \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) such that \\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\)

\n

two-party protocols for ECDSA signing use multiplicative sharing of \\( x \\) and of \\( k \\). That is, the parties hold \\(x_1\\), \\(x_2\\) such that \\(x_1 \\cdot x_2 = x \\bmod q\\), and in each signing operation they generate \\(k_1\\), \\(k_2\\) such that \\(k_1 \\cdot k_2 = k \\bmod q\\). This enables them to easily compute \\(k^{-1}\\) since each party can locally compute \\(k_i^{\\prime} = k_i^{-1} \\bmod q\\), and then \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) are multiplicative shares of \\(k^{-1}\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\(P_1\\) can compute \\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\) and \\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\( P_2 \\) can compute \\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\), which will be an encryption of

\n

\"paillier

\n

However, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\(k_1^{-1}\\) when the second party only has \\(R_1 = k_1 \\cdot G\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.

\n

their results

[WIP]

\n

references

\n"},{"title":"go similar concepts comparison","date":"2023-03-05T02:44:50.000Z","_content":"\n## struct{} & struct{}{}\n`struct` is a keyword in Go. It is used to define struct types, which is a sequence of named elements.\n\nFor example:\n```go\ntype Person struct {\n Name string\n Age int\n}\n```\nThe `struct{}` is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type `struct{}`.\n\n`struct{}{}` on the other hand is a [composite literal](https://go.dev/ref/spec#Composite_literals), it constructs a value of type `struct{}`. A composite literal constructs values for types such as `structs`, `arrays`, `maps` and `slices`. Its syntax is the type followed by the elements in braces. Since the \"empty\" struct (struct{}) has no fields, the elements list is also empty:\n\n struct{} {}\n| ^ | ^\n type empty element list\nAs an example let's create a \"set\" in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.\n\nA map with string elements:\n```go\nvar set map[string]struct{}\n// Initialize the set\nset = make(map[string]struct{})\n\n// Add some values to the set:\nset[\"red\"] = struct{}{}\nset[\"blue\"] = struct{}{}\n\n// Check if a value is in the map:\n_, ok := set[\"red\"]\nfmt.Println(\"Is red in the map?\", ok)\n_, ok = set[\"green\"]\nfmt.Println(\"Is green in the map?\", ok)\n```","source":"_posts/golang/go-similar-concepts-comparison.md","raw":"---\ntitle: go similar concepts comparison\ndate: 2023-03-05 10:44:50\ntags: [golang]\n---\n\n## struct{} & struct{}{}\n`struct` is a keyword in Go. It is used to define struct types, which is a sequence of named elements.\n\nFor example:\n```go\ntype Person struct {\n Name string\n Age int\n}\n```\nThe `struct{}` is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type `struct{}`.\n\n`struct{}{}` on the other hand is a [composite literal](https://go.dev/ref/spec#Composite_literals), it constructs a value of type `struct{}`. A composite literal constructs values for types such as `structs`, `arrays`, `maps` and `slices`. Its syntax is the type followed by the elements in braces. Since the \"empty\" struct (struct{}) has no fields, the elements list is also empty:\n\n struct{} {}\n| ^ | ^\n type empty element list\nAs an example let's create a \"set\" in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.\n\nA map with string elements:\n```go\nvar set map[string]struct{}\n// Initialize the set\nset = make(map[string]struct{})\n\n// Add some values to the set:\nset[\"red\"] = struct{}{}\nset[\"blue\"] = struct{}{}\n\n// Check if a value is in the map:\n_, ok := set[\"red\"]\nfmt.Println(\"Is red in the map?\", ok)\n_, ok = set[\"green\"]\nfmt.Println(\"Is green in the map?\", ok)\n```","slug":"golang/go-similar-concepts-comparison","published":1,"updated":"2023-06-18T07:07:22.116Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzo000kofsj3zwhcohs","content":"

struct{} & struct{}{}

struct is a keyword in Go. It is used to define struct types, which is a sequence of named elements.

\n

For example:

\n
1
2
3
4
type Person struct {
    Name string
    Age  int
}
\n

The struct{} is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type struct{}.

\n

struct{}{} on the other hand is a composite literal, it constructs a value of type struct{}. A composite literal constructs values for types such as structs, arrays, maps and slices. Its syntax is the type followed by the elements in braces. Since the “empty” struct (struct{}) has no fields, the elements list is also empty:

\n

struct{} {}
| ^ | ^
type empty element list
As an example let’s create a “set” in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.

\n

A map with string elements:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
var set map[string]struct{}
// Initialize the set
set = make(map[string]struct{})

// Add some values to the set:
set["red"] = struct{}{}
set["blue"] = struct{}{}

// Check if a value is in the map:
_, ok := set["red"]
fmt.Println("Is red in the map?", ok)
_, ok = set["green"]
fmt.Println("Is green in the map?", ok)
","site":{"data":{}},"excerpt":"","more":"

struct{} & struct{}{}

struct is a keyword in Go. It is used to define struct types, which is a sequence of named elements.

\n

For example:

\n
1
2
3
4
type Person struct {
    Name string
    Age  int
}
\n

The struct{} is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type struct{}.

\n

struct{}{} on the other hand is a composite literal, it constructs a value of type struct{}. A composite literal constructs values for types such as structs, arrays, maps and slices. Its syntax is the type followed by the elements in braces. Since the “empty” struct (struct{}) has no fields, the elements list is also empty:

\n

struct{} {}
| ^ | ^
type empty element list
As an example let’s create a “set” in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.

\n

A map with string elements:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
var set map[string]struct{}
// Initialize the set
set = make(map[string]struct{})

// Add some values to the set:
set["red"] = struct{}{}
set["blue"] = struct{}{}

// Check if a value is in the map:
_, ok := set["red"]
fmt.Println("Is red in the map?", ok)
_, ok = set["green"]
fmt.Println("Is green in the map?", ok)
"},{"title":"rust resource","date":"2022-09-27T14:04:38.000Z","_content":"\n## learning resources\n- [the book](https://doc.rust-lang.org/book/)\n- [cargo book](https://doc.rust-lang.org/cargo/index.html)\n- [the rust performance book](https://nnethercote.github.io/perf-book/title-page.html)\n- [rust design patterns](https://rust-unofficial.github.io/patterns/intro.html)\n- [the rust RFC book](https://rust-lang.github.io/rfcs/)\n- [the rustdoc book](https://doc.rust-lang.org/rustdoc/what-is-rustdoc.html)\n- [rustnomicon](https://doc.rust-lang.org/nomicon/intro.html)\n- [the rust reference](https://doc.rust-lang.org/reference/introduction.html)\n- [rust by example](https://doc.rust-lang.org/rust-by-example/index.html)\n- [async programming in rust](https://rust-lang.github.io/async-book/01_getting_started/01_chapter.html)\n- [rust compiler development guide](https://rustc-dev-guide.rust-lang.org/about-this-guide.html)\n- [the little book of rust macros](https://veykril.github.io/tlborm/)\n\n## video resources\n- [youtube channel](https://www.youtube.com/@jonhoo)\n\n## books\n- [atomics and locks](https://marabos.nl/atomics/)\n- [data structure & algorithm](https://github.com/QMHTMY/RustBook/blob/main/books/rust-book-zh-cn-shieber.pdf)","source":"_posts/rust/rust-01-resource.md","raw":"---\ntitle: rust resource\ndate: 2022-09-27 22:04:38\ntags: [rust]\n---\n\n## learning resources\n- [the book](https://doc.rust-lang.org/book/)\n- [cargo book](https://doc.rust-lang.org/cargo/index.html)\n- [the rust performance book](https://nnethercote.github.io/perf-book/title-page.html)\n- [rust design patterns](https://rust-unofficial.github.io/patterns/intro.html)\n- [the rust RFC book](https://rust-lang.github.io/rfcs/)\n- [the rustdoc book](https://doc.rust-lang.org/rustdoc/what-is-rustdoc.html)\n- [rustnomicon](https://doc.rust-lang.org/nomicon/intro.html)\n- [the rust reference](https://doc.rust-lang.org/reference/introduction.html)\n- [rust by example](https://doc.rust-lang.org/rust-by-example/index.html)\n- [async programming in rust](https://rust-lang.github.io/async-book/01_getting_started/01_chapter.html)\n- [rust compiler development guide](https://rustc-dev-guide.rust-lang.org/about-this-guide.html)\n- [the little book of rust macros](https://veykril.github.io/tlborm/)\n\n## video resources\n- [youtube channel](https://www.youtube.com/@jonhoo)\n\n## books\n- [atomics and locks](https://marabos.nl/atomics/)\n- [data structure & algorithm](https://github.com/QMHTMY/RustBook/blob/main/books/rust-book-zh-cn-shieber.pdf)","slug":"rust/rust-01-resource","published":1,"updated":"2023-06-29T04:06:05.747Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzo000mofsj1fzd99u4","content":"

learning resources

\n

video resources

\n

books

\n","site":{"data":{}},"excerpt":"","more":"

learning resources

\n

video resources

\n

books

\n"},{"title":"rust basics","date":"2022-10-04T07:55:04.000Z","_content":"\n## frequently used cmd\n```\nrustc [filename].rs\ncargo new [project_name]\ncargo build [--release]\ncargo run [--release]\ncargo check # check whether compile success, no executible output\n```\n\n## data type\n\n### integer\n- i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc\n- isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.\n- 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）\n\n| Number Literals | Example |\n| ----------- | ----------- |\n| Decimal | 98_222 |\n| Hex | 0xff |\n| Octal | 0o77 |\n| Binary | 0b1111_0000 |\n| Byte(u8 only) | b'A' |\n\n### Tuple\n- The length of Tuple is fixed, and the length cannot be changed once declared\n```rust\nfn main() {\n // tuple could be declared as mut\n let mut tuple_1 = (\"Hello\", 39, \"Years\");\n let tuple_2:(i32, &str ) = (1983, \"since.\");\n tuple_1.0 = \"Hi\";\n println!(\"{} {} {}\", tuple_1.0, tuple_1.1, tuple_1.2);\n // destructure\n let (a,b) = tuple_2;\n println!(\"{} {}\", a, b);\n}\n```\n\n### array\n- arrays in Rust have a fixed length.\n- Vector is similar to an array, it is provided by the standard library, and its length can be changed\n\n```rust\nfn main() {\n\n let arr_test:[u8; 3] = [1,2,3];\n println!(\"Number is {},{},{}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [\"I\",\"love\",\"you\"];\n println!(\"You said : {} {} {}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [1;3]; \n println!(\"Call Num : {}&{}&{}\", arr_test[0],arr_test[1],arr_test[2]);\n}\n```\n\n\n\n### String\n- Basic data types are stored on the stack, but the String type is stored on the heap\n```rust\nlet s = String::from(\"hello\");\n```\n- push_str(): append a str slice a string\n- push(): appends a single character to a String\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n}\n```\n- + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice\n- String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len\n- String iteration\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n\n for i in data.bytes() {\n ///\n }\n\n for i in data.chars() {\n ///\n }\n}\n```\n\n### Vector\n- Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.\n```rust\nfn main() {\n let vec: Vec = Vec::new();\n let vec2: Vec = vec![3,4，5] // create vector by macro\n for i in vec2 {\n println!(\"Vector value is : {}\", i);\n }\n}\n```\n\n### HashMap\n- HashMap is not preloaded, so it needs to be included `use std::collections::HashMap`\n```rust\nuse std::collections::HashMap;\nfn main() {\n let keys = vec![\"andy\".to_string(), \"cliff\".to_string()] ;\n let ages = vec![38, 26];\n let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();\n println!(\"{:?}\", map); /// print {\"andy\": 38, \"cliff\": 26}\n}\n```\n#### HashMap ownership\n- For types that implement the Copy trait (such as i32), the value will be copied into the HashMap\n- For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap\n- If a reference to a value is inserted into the HashMap, the value itself does not move\n\n#### HashMap iteration\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n println!(\"{:?}\", &map);\n for (k, v) in map {\n println!(\"{} age {}\", k, v);\n } /// cliff age 26\n /// andy age 36\n}\n```\n#### update\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n\n let result = map.entry(\"bob\".to_string());\n println!(\"{:?}\", result); /// Entry(VacantEntry(\"bob\"))\n\n let result = map.entry(\"andy\".to_string());\n println!(\"{:?}\", result); /// Entry(OccupiedEntry { key: \"andy\", value: 36, .. })\n\n map.entry(\"bob\".to_string()).or_insert(28);\n map.entry(\"cliff\".to_string()).or_insert(0);\n}\n```\n\n## control flow\n- if\n```rust\nfn main() {\n let condition = 1;\n let x = if condition == 1 { \"A\" } else { \"B\" };\n println!(\"Result x = {}\" , x) ;\n}\n```\n- loop\n```rust\nfn main() {\n let mut condition = 0;\n\n let result = 'outer: loop { // 'outer is label\n 'inner: loop {\n condition += 1;\n if 3 == condition {\n break 'outer 3 * condition; // break outer loop\n }\n }\n };\n println!(\"Loop result is : {}\", result); /// Loop result is : 9\n}\n\n```\n- rot\n```rust\nfn main() {\n let arr = [3,2,3];\n for num in arr.iter() {\n println!(\"For value is {}\", num);\n }\n}\n```\n\n## Range iterator\n- Range\n```rust\nfn main() {\n for number in (1..=3) {\n println!(\"Number A is {}\", number ); /// 1,2,3\n }\n \n for number in (1..=3).rev() { /// rev means reverse,\n println!(\"Number B is {}\", number ); /// 3,2,1\n }\n}\n\n```\n\n## struct\n- If struct is declared mutable then all fields in the instance are mutable\n### tuple struct\n```rust\nstruct Color(i32,i32,i32);\nlet black = Color(0,0,0);\n```\n### Unit-Like struct\n```rust\nstruct Man {};\n```\n### struct method\n```rust\n\nfn main() {\n let rec = Rectangle {\n width: 30,\n height: 50,\n };\n\n let result = rec.area(); \n println!(\"rectangle：{:?}，area is：{}\", rec, result);\n}\n\n\n#[derive(Debug)]\nstruct Rectangle {\n width: u32,\n height: u32,\n}\n\nimpl Rectangle {\n fn area(&self) -> u32{\n self.width * self.height\n }\n}\n\n```\n### associative func（similar to static method）\n- You can define a function that does not take `self` as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to `String::from()`\n```rust\nimpl Rectangle {\n fn create_square(width: u32) -> Rectangle {\n Rectangle {\n width,\n height: width,\n }\n }\n}\n```\n \n## enum\n```rust\nenum Ip {\n V4,\n V6,\n}\n\nenum IpAddr {\n V4(String),\n V6(String),\n}\n``` \n\n## match\n- match must exhaust all possibilities\n- If there are too many matchings, you can also use \"_\" for wildcarding, but note that \"_\" must be placed at the end\n```rust\nenum Color {\n Red,\n Yellow,\n Blue,\n}\nenum ColorWithVal {\n Red(u8,u8,u8),\n Yellow(u8,u8,u8),\n Blue(u8,u8,u8),\n}\nfn main(){\n let colour = Color::Blue;\n match colour {\n Color::Red => {\n println!(\"Red colour.\");\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n\n let colour = ColorWithVal::Red(222,111,22);\n match colour {\n ColorWithVal::Red(r,g,b) => {\n println!(\"Red colour. {},{},{}\", r,g,b);\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n}\n```\n\n## if let\n```rust\nfn main(){\n let colour = Color::Red(Some(222),Some(222),Some(222));\n\n if let Color::Red(r,g,b) = colour {\n println!(\"Red colour. {:?},{:?},{:?}\", r,g,b);\n } else {\n println!(\"Other colour.\");\n }\n}\n```\n\n## Result\n- Recoverable err via Result, non-recoverable via panic!\n- upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program\n- You can set panic = 'abort' in Cargo.toml to terminate the cleaning of the call stack\n```rust\n[profile.release]\npanic='abort'\n```\n- RUST_BACKTRACE = 1 prints detailed error messages in the stack\n```rust\nuse std::fs::File;\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => panic!(\"file not found {:?} \", error),\n };\n}\n```\n\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => {\n match error.kind() {\n ErrorKind::NotFound => {\n match File::create(\"hello.txt\") {\n Ok(file) => {\n file\n },\n Err(err) => {\n panic!(\"file create error:{:?}\", &err);\n },\n }\n },\n oe => panic!(\"other error {:?}\", oe),\n }\n } ,\n };\n}\n```\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let file = File::open(\"hello.txt\").unwrap_or_else(|err| {\n if err.kind() == ErrorKind::NotFound {\n File::create(\"hello.txt\").unwrap_or_else(|err|{\n panic!(\"error：{:?}\", err);\n })\n }else{\n panic!(\"other error：{:?}\", err);\n }\n });\n}\n```\n### unwrap && expect\n- If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.\n- expect can specify what the error message is, which is easier to debug\n\n### The question mark operator, ?\nWhen writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.\n```rust\nlet mut file = File::create(\"my_best_friends.txt\")?;\n```\n\n## generic\n```rust\n#[derive(Debug)]\nstruct Point {\n x : T,\n y : U,\n}\nimpl Point {\n fn mixup(self, other: Point) -> Point {\n Point{x: self.x , y: other.y, }\n }\n}\n```\n\n## trait\n### definition\n```rust\npub trait Summary {\n fn summarize(&self) -> String {\n \"... more\".to_string() /// default \n }\n}\npub struct Tweet {\n user_name :String,\n replay_count :u32,\n like_count :u32,\n}\nimpl Tweet {\n fn like(&mut self) {\n self.like_count += 1;\n }\n}\n\nimpl Summary for Tweet {\n fn summarize(&self) -> String {\n format!(\"{} like count :{} , replay count :{}\", &self.user_name, &self.replay_count, &self.like_count)\n }\n}\n```\n\n### trait as arguments\n```rust\nfn notify_msg (info: T) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: impl Summary + Display) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: T) \nwhere \n T: Summary + Display,\n{\n println!(\"summary : {}\", info.summarize() );\n println!(\"display implement info : {}\", info);\n}\n```\n### trait as return\n- impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported\n\n## references\n- [the rust programming language](https://doc.rust-lang.org/book/)\n- [mooc course](https://time.geekbang.org/course/intro/100060601?tab=catalog)\n- [bilibili tutorial](https://www.bilibili.com/video/BV1hp4y1k7SV?p=50&spm_id_from=pageDriver)\n- [jianshu notes](https://www.jianshu.com/p/30d917790298)","source":"_posts/rust/rust-02-basics.md","raw":"---\ntitle: rust basics\ndate: 2022-10-04 15:55:04\ntags: [rust]\n---\n\n## frequently used cmd\n```\nrustc [filename].rs\ncargo new [project_name]\ncargo build [--release]\ncargo run [--release]\ncargo check # check whether compile success, no executible output\n```\n\n## data type\n\n### integer\n- i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc\n- isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.\n- 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）\n\n| Number Literals | Example |\n| ----------- | ----------- |\n| Decimal | 98_222 |\n| Hex | 0xff |\n| Octal | 0o77 |\n| Binary | 0b1111_0000 |\n| Byte(u8 only) | b'A' |\n\n### Tuple\n- The length of Tuple is fixed, and the length cannot be changed once declared\n```rust\nfn main() {\n // tuple could be declared as mut\n let mut tuple_1 = (\"Hello\", 39, \"Years\");\n let tuple_2:(i32, &str ) = (1983, \"since.\");\n tuple_1.0 = \"Hi\";\n println!(\"{} {} {}\", tuple_1.0, tuple_1.1, tuple_1.2);\n // destructure\n let (a,b) = tuple_2;\n println!(\"{} {}\", a, b);\n}\n```\n\n### array\n- arrays in Rust have a fixed length.\n- Vector is similar to an array, it is provided by the standard library, and its length can be changed\n\n```rust\nfn main() {\n\n let arr_test:[u8; 3] = [1,2,3];\n println!(\"Number is {},{},{}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [\"I\",\"love\",\"you\"];\n println!(\"You said : {} {} {}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [1;3]; \n println!(\"Call Num : {}&{}&{}\", arr_test[0],arr_test[1],arr_test[2]);\n}\n```\n\n\n\n### String\n- Basic data types are stored on the stack, but the String type is stored on the heap\n```rust\nlet s = String::from(\"hello\");\n```\n- push_str(): append a str slice a string\n- push(): appends a single character to a String\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n}\n```\n- + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice\n- String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len\n- String iteration\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n\n for i in data.bytes() {\n ///\n }\n\n for i in data.chars() {\n ///\n }\n}\n```\n\n### Vector\n- Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.\n```rust\nfn main() {\n let vec: Vec = Vec::new();\n let vec2: Vec = vec![3,4，5] // create vector by macro\n for i in vec2 {\n println!(\"Vector value is : {}\", i);\n }\n}\n```\n\n### HashMap\n- HashMap is not preloaded, so it needs to be included `use std::collections::HashMap`\n```rust\nuse std::collections::HashMap;\nfn main() {\n let keys = vec![\"andy\".to_string(), \"cliff\".to_string()] ;\n let ages = vec![38, 26];\n let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();\n println!(\"{:?}\", map); /// print {\"andy\": 38, \"cliff\": 26}\n}\n```\n#### HashMap ownership\n- For types that implement the Copy trait (such as i32), the value will be copied into the HashMap\n- For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap\n- If a reference to a value is inserted into the HashMap, the value itself does not move\n\n#### HashMap iteration\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n println!(\"{:?}\", &map);\n for (k, v) in map {\n println!(\"{} age {}\", k, v);\n } /// cliff age 26\n /// andy age 36\n}\n```\n#### update\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n\n let result = map.entry(\"bob\".to_string());\n println!(\"{:?}\", result); /// Entry(VacantEntry(\"bob\"))\n\n let result = map.entry(\"andy\".to_string());\n println!(\"{:?}\", result); /// Entry(OccupiedEntry { key: \"andy\", value: 36, .. })\n\n map.entry(\"bob\".to_string()).or_insert(28);\n map.entry(\"cliff\".to_string()).or_insert(0);\n}\n```\n\n## control flow\n- if\n```rust\nfn main() {\n let condition = 1;\n let x = if condition == 1 { \"A\" } else { \"B\" };\n println!(\"Result x = {}\" , x) ;\n}\n```\n- loop\n```rust\nfn main() {\n let mut condition = 0;\n\n let result = 'outer: loop { // 'outer is label\n 'inner: loop {\n condition += 1;\n if 3 == condition {\n break 'outer 3 * condition; // break outer loop\n }\n }\n };\n println!(\"Loop result is : {}\", result); /// Loop result is : 9\n}\n\n```\n- rot\n```rust\nfn main() {\n let arr = [3,2,3];\n for num in arr.iter() {\n println!(\"For value is {}\", num);\n }\n}\n```\n\n## Range iterator\n- Range\n```rust\nfn main() {\n for number in (1..=3) {\n println!(\"Number A is {}\", number ); /// 1,2,3\n }\n \n for number in (1..=3).rev() { /// rev means reverse,\n println!(\"Number B is {}\", number ); /// 3,2,1\n }\n}\n\n```\n\n## struct\n- If struct is declared mutable then all fields in the instance are mutable\n### tuple struct\n```rust\nstruct Color(i32,i32,i32);\nlet black = Color(0,0,0);\n```\n### Unit-Like struct\n```rust\nstruct Man {};\n```\n### struct method\n```rust\n\nfn main() {\n let rec = Rectangle {\n width: 30,\n height: 50,\n };\n\n let result = rec.area(); \n println!(\"rectangle：{:?}，area is：{}\", rec, result);\n}\n\n\n#[derive(Debug)]\nstruct Rectangle {\n width: u32,\n height: u32,\n}\n\nimpl Rectangle {\n fn area(&self) -> u32{\n self.width * self.height\n }\n}\n\n```\n### associative func（similar to static method）\n- You can define a function that does not take `self` as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to `String::from()`\n```rust\nimpl Rectangle {\n fn create_square(width: u32) -> Rectangle {\n Rectangle {\n width,\n height: width,\n }\n }\n}\n```\n \n## enum\n```rust\nenum Ip {\n V4,\n V6,\n}\n\nenum IpAddr {\n V4(String),\n V6(String),\n}\n``` \n\n## match\n- match must exhaust all possibilities\n- If there are too many matchings, you can also use \"_\" for wildcarding, but note that \"_\" must be placed at the end\n```rust\nenum Color {\n Red,\n Yellow,\n Blue,\n}\nenum ColorWithVal {\n Red(u8,u8,u8),\n Yellow(u8,u8,u8),\n Blue(u8,u8,u8),\n}\nfn main(){\n let colour = Color::Blue;\n match colour {\n Color::Red => {\n println!(\"Red colour.\");\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n\n let colour = ColorWithVal::Red(222,111,22);\n match colour {\n ColorWithVal::Red(r,g,b) => {\n println!(\"Red colour. {},{},{}\", r,g,b);\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n}\n```\n\n## if let\n```rust\nfn main(){\n let colour = Color::Red(Some(222),Some(222),Some(222));\n\n if let Color::Red(r,g,b) = colour {\n println!(\"Red colour. {:?},{:?},{:?}\", r,g,b);\n } else {\n println!(\"Other colour.\");\n }\n}\n```\n\n## Result\n- Recoverable err via Result, non-recoverable via panic!\n- upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program\n- You can set panic = 'abort' in Cargo.toml to terminate the cleaning of the call stack\n```rust\n[profile.release]\npanic='abort'\n```\n- RUST_BACKTRACE = 1 prints detailed error messages in the stack\n```rust\nuse std::fs::File;\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => panic!(\"file not found {:?} \", error),\n };\n}\n```\n\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => {\n match error.kind() {\n ErrorKind::NotFound => {\n match File::create(\"hello.txt\") {\n Ok(file) => {\n file\n },\n Err(err) => {\n panic!(\"file create error:{:?}\", &err);\n },\n }\n },\n oe => panic!(\"other error {:?}\", oe),\n }\n } ,\n };\n}\n```\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let file = File::open(\"hello.txt\").unwrap_or_else(|err| {\n if err.kind() == ErrorKind::NotFound {\n File::create(\"hello.txt\").unwrap_or_else(|err|{\n panic!(\"error：{:?}\", err);\n })\n }else{\n panic!(\"other error：{:?}\", err);\n }\n });\n}\n```\n### unwrap && expect\n- If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.\n- expect can specify what the error message is, which is easier to debug\n\n### The question mark operator, ?\nWhen writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.\n```rust\nlet mut file = File::create(\"my_best_friends.txt\")?;\n```\n\n## generic\n```rust\n#[derive(Debug)]\nstruct Point {\n x : T,\n y : U,\n}\nimpl Point {\n fn mixup(self, other: Point) -> Point {\n Point{x: self.x , y: other.y, }\n }\n}\n```\n\n## trait\n### definition\n```rust\npub trait Summary {\n fn summarize(&self) -> String {\n \"... more\".to_string() /// default \n }\n}\npub struct Tweet {\n user_name :String,\n replay_count :u32,\n like_count :u32,\n}\nimpl Tweet {\n fn like(&mut self) {\n self.like_count += 1;\n }\n}\n\nimpl Summary for Tweet {\n fn summarize(&self) -> String {\n format!(\"{} like count :{} , replay count :{}\", &self.user_name, &self.replay_count, &self.like_count)\n }\n}\n```\n\n### trait as arguments\n```rust\nfn notify_msg (info: T) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: impl Summary + Display) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: T) \nwhere \n T: Summary + Display,\n{\n println!(\"summary : {}\", info.summarize() );\n println!(\"display implement info : {}\", info);\n}\n```\n### trait as return\n- impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported\n\n## references\n- [the rust programming language](https://doc.rust-lang.org/book/)\n- [mooc course](https://time.geekbang.org/course/intro/100060601?tab=catalog)\n- [bilibili tutorial](https://www.bilibili.com/video/BV1hp4y1k7SV?p=50&spm_id_from=pageDriver)\n- [jianshu notes](https://www.jianshu.com/p/30d917790298)","slug":"rust/rust-02-basics","published":1,"updated":"2023-05-01T09:07:13.124Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzo000oofsj3jfs99sr","content":"

frequently used cmd

1
2
3
4
5
rustc [filename].rs
cargo new [project_name]
cargo build [--release]
cargo run [--release]
cargo check # check whether compile success, no executible output
\n\n

data type

integer

    \n
  • i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc
    • \n
  • isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.
    • \n
  • 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）
    • \n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Number LiteralsExample
Decimal98_222
Hex0xff
Octal0o77
Binary0b1111_0000
Byte(u8 only)b’A’
\n

Tuple

    \n
  • The length of Tuple is fixed, and the length cannot be changed once declared
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
        // tuple could be declared as mut
        let mut tuple_1 = ("Hello", 39, "Years");
        let tuple_2:(i32, &str ) = (1983, "since.");
        tuple_1.0 = "Hi";
        println!("{} {} {}", tuple_1.0, tuple_1.1, tuple_1.2);
        // destructure
        let (a,b) = tuple_2;
        println!("{} {}", a, b);
    }
    • \n
\n

array

    \n
  • arrays in Rust have a fixed length.
    • \n
  • Vector is similar to an array, it is provided by the standard library, and its length can be changed
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
fn main() {

    let arr_test:[u8; 3] = [1,2,3];
    println!("Number is {},{},{}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = ["I","love","you"];
    println!("You said : {} {} {}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = [1;3]; 
    println!("Call Num : {}&{}&{}", arr_test[0],arr_test[1],arr_test[2]);
}
\n\n\n\n

String

    \n
  • Basic data types are stored on the stack, but the String type is stored on the heap
    1
    let s = String::from("hello");
    • \n
  • push_str(): append a str slice a string
    • \n
  • push(): appends a single character to a String
    1
    2
    3
    4
    5
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');
    }
    • \n
  • + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice
    • \n
  • String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len
    • \n
  • String iteration
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');

        for i in data.bytes() {
            ///
        }

        for i in data.chars() {
            ///
        }
    }
    • \n
\n

Vector

    \n
  • Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.
    1
    2
    3
    4
    5
    6
    7
    fn main() {
        let vec: Vec<u16> = Vec::new();
        let vec2: Vec<i32> = vec![3,4，5] // create vector by macro
        for i in vec2 {
            println!("Vector value is : {}", i);
        }
    }
    • \n
\n

HashMap

    \n
  • HashMap is not preloaded, so it needs to be included use std::collections::HashMap
    1
    2
    3
    4
    5
    6
    7
    use std::collections::HashMap;
    fn main() {
        let keys = vec!["andy".to_string(), "cliff".to_string()] ;
        let ages = vec![38, 26];
        let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();
        println!("{:?}", map); /// print {"andy": 38, "cliff": 26}
    }
    • \n
\n

HashMap ownership

    \n
  • For types that implement the Copy trait (such as i32), the value will be copied into the HashMap
    • \n
  • For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap
    • \n
  • If a reference to a value is inserted into the HashMap, the value itself does not move
    • \n
\n

HashMap iteration

1
2
3
4
5
6
7
8
9
10
11
12
13
14
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);
    println!("{:?}", &map);
    for (k, v) in map {
        println!("{} age {}", k, v);
    } /// cliff age 26
      /// andy age 36
}
\n

update

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);

    let result = map.entry("bob".to_string());
    println!("{:?}", result); /// Entry(VacantEntry("bob"))

    let result = map.entry("andy".to_string());
    println!("{:?}", result); /// Entry(OccupiedEntry { key: "andy", value: 36, .. })

    map.entry("bob".to_string()).or_insert(28);
    map.entry("cliff".to_string()).or_insert(0);
}
\n\n

control flow

    \n
  • if
    1
    2
    3
    4
    5
    fn main() {
        let condition = 1;
        let x = if condition == 1 { "A" } else { "B" };
        println!("Result x = {}" , x) ;
    }
    • \n
  • loop
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    fn main() {
        let mut condition = 0;

        let result = 'outer: loop {  // 'outer is label
            'inner: loop {
                condition += 1;
                if 3 == condition {
                    break 'outer 3 * condition; // break outer loop
                }
            }
        };
        println!("Loop result is : {}", result); /// Loop result is : 9
    }

    • \n
  • rot
    1
    2
    3
    4
    5
    6
    fn main() {
        let arr = [3,2,3];
        for num in arr.iter() {
            println!("For value is {}", num);
        }
    }
    • \n
\n

Range iterator

    \n
  • Range
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
         for number in (1..=3) {
            println!("Number A is {}", number ); /// 1,2,3
        }
     
        for number in (1..=3).rev() { /// rev means reverse,
            println!("Number B is {}", number ); /// 3,2,1
        }
    }

    • \n
\n

struct

    \n
  • If struct is declared mutable then all fields in the instance are mutable
    • \n
\n

tuple struct

1
2
struct Color(i32,i32,i32);
let black = Color(0,0,0);
\n

Unit-Like struct

1
struct Man {};
\n

struct method

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

fn main() {
    let rec = Rectangle {
        width: 30,
        height: 50,
    };

    let result = rec.area(); 
    println!("rectangle：{:?}，area is：{}", rec, result);
}


#[derive(Debug)]
struct Rectangle {
    width: u32,
    height: u32,
}

impl Rectangle {
    fn area(&self) -> u32{
        self.width * self.height
    }
}

\n

associative func（similar to static method）

    \n
  • You can define a function that does not take self as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to String::from()
    1
    2
    3
    4
    5
    6
    7
    8
    impl Rectangle {
        fn create_square(width: u32) -> Rectangle {
            Rectangle {
                width,
                height: width,
            }
        }
    }
    • \n
\n

enum

1
2
3
4
5
6
7
8
9
enum Ip {
    V4,
    V6,
}

enum IpAddr {
    V4(String),
    V6(String),
}
\n\n

match

    \n
  • match must exhaust all possibilities
    • \n
  • If there are too many matchings, you can also use ““ for wildcarding, but note that ““ must be placed at the end
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    enum Color {
        Red,
        Yellow,
        Blue,
    }
    enum ColorWithVal {
        Red(u8,u8,u8),
        Yellow(u8,u8,u8),
        Blue(u8,u8,u8),
    }
    fn main(){
        let colour = Color::Blue;
        match colour {
            Color::Red => {
                println!("Red colour.");
            },
            _ => {
                println!("Other colour.");
            }
        }

        let colour = ColorWithVal::Red(222,111,22);
        match colour {
            ColorWithVal::Red(r,g,b) => {
                println!("Red colour. {},{},{}", r,g,b);
            },
            _ => {
                println!("Other colour.");
            }
        }
    }
    • \n
\n

if let

1
2
3
4
5
6
7
8
9
fn main(){
    let colour = Color::Red(Some(222),Some(222),Some(222));

    if let Color::Red(r,g,b) = colour {
        println!("Red colour. {:?},{:?},{:?}", r,g,b);
    } else {
        println!("Other colour.");
    }
}
\n\n

Result<T,E>

    \n
  • Recoverable err via Result<T,E>, non-recoverable via panic!
    • \n
  • upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program
    • \n
  • You can set panic = ‘abort’ in Cargo.toml to terminate the cleaning of the call stack
    1
    2
    [profile.release]
    panic='abort'
    • \n
  • RUST_BACKTRACE = 1 prints detailed error messages in the stack
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::fs::File;
    fn main() { 
        let fp = File::open("hello.txt");
        let file = match fp {
            Ok(file)=> {
                file
            },
            Err(error) => panic!("file not found {:?} ", error),
        };
    }
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
use std::{fs::File, io::ErrorKind};
fn main() { 
    let fp = File::open("hello.txt");
    let file = match fp {
        Ok(file)=> {
            file
        },
        Err(error) => {
            match error.kind() {
                ErrorKind::NotFound => {
                    match File::create("hello.txt") {
                        Ok(file) => {
                            file
                        },
                        Err(err) => {
                            panic!("file create error:{:?}", &err);
                        },
                    }
                },
                oe => panic!("other error {:?}", oe),
            }
        } ,
    };
}
\n
1
2
3
4
5
6
7
8
9
10
11
12
use std::{fs::File, io::ErrorKind};
fn main() { 
    let file = File::open("hello.txt").unwrap_or_else(|err| {
        if err.kind() == ErrorKind::NotFound {
            File::create("hello.txt").unwrap_or_else(|err|{
                panic!("error：{:?}", err);
            })
        }else{
            panic!("other error：{:?}", err);
        }
    });
}
\n

unwrap && expect

    \n
  • If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.
    • \n
  • expect can specify what the error message is, which is easier to debug
    • \n
\n

The question mark operator, ?

When writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.

\n
1
let mut file = File::create("my_best_friends.txt")?;
\n\n

generic

1
2
3
4
5
6
7
8
9
10
#[derive(Debug)]
struct Point<T, U>  {
   x : T,
   y : U,
}
impl <T, U> Point<T, U> {
    fn mixup<V, W>(self, other: Point<V, W>) -> Point<T, W> {
        Point{x: self.x , y: other.y, }
    }
}
\n\n

trait

definition

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
pub trait Summary {
    fn summarize(&self) -> String {
         "... more".to_string() /// default 
    }
}
pub struct Tweet {
    user_name :String,
    replay_count :u32,
    like_count :u32,
}
impl Tweet {
    fn like(&mut self) {
        self.like_count += 1;
    }
}

impl Summary for Tweet {
    fn summarize(&self) -> String {
        format!("{} like count :{} , replay count :{}", &self.user_name, &self.replay_count, &self.like_count)
    }
}
\n\n

trait as arguments

1
2
3
4
5
6
7
8
9
10
11
12
13
fn notify_msg <T:Summary> (info: T) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg (info: impl Summary + Display) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg <T> (info: T) 
where 
    T: Summary + Display,
{
    println!("summary : {}", info.summarize() );
    println!("display implement info : {}", info);
}
\n

trait as return

    \n
  • impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported
    • \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"

frequently used cmd

1
2
3
4
5
rustc [filename].rs
cargo new [project_name]
cargo build [--release]
cargo run [--release]
cargo check # check whether compile success, no executible output
\n\n

data type

integer

    \n
  • i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc
    • \n
  • isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.
    • \n
  • 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）
    • \n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Number LiteralsExample
Decimal98_222
Hex0xff
Octal0o77
Binary0b1111_0000
Byte(u8 only)b’A’
\n

Tuple

    \n
  • The length of Tuple is fixed, and the length cannot be changed once declared
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
        // tuple could be declared as mut
        let mut tuple_1 = ("Hello", 39, "Years");
        let tuple_2:(i32, &str ) = (1983, "since.");
        tuple_1.0 = "Hi";
        println!("{} {} {}", tuple_1.0, tuple_1.1, tuple_1.2);
        // destructure
        let (a,b) = tuple_2;
        println!("{} {}", a, b);
    }
    • \n
\n

array

    \n
  • arrays in Rust have a fixed length.
    • \n
  • Vector is similar to an array, it is provided by the standard library, and its length can be changed
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
fn main() {

    let arr_test:[u8; 3] = [1,2,3];
    println!("Number is {},{},{}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = ["I","love","you"];
    println!("You said : {} {} {}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = [1;3]; 
    println!("Call Num : {}&{}&{}", arr_test[0],arr_test[1],arr_test[2]);
}
\n\n\n\n

String

    \n
  • Basic data types are stored on the stack, but the String type is stored on the heap
    1
    let s = String::from("hello");
    • \n
  • push_str(): append a str slice a string
    • \n
  • push(): appends a single character to a String
    1
    2
    3
    4
    5
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');
    }
    • \n
  • + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice
    • \n
  • String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len
    • \n
  • String iteration
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');

        for i in data.bytes() {
            ///
        }

        for i in data.chars() {
            ///
        }
    }
    • \n
\n

Vector

    \n
  • Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.
    1
    2
    3
    4
    5
    6
    7
    fn main() {
        let vec: Vec<u16> = Vec::new();
        let vec2: Vec<i32> = vec![3,4，5] // create vector by macro
        for i in vec2 {
            println!("Vector value is : {}", i);
        }
    }
    • \n
\n

HashMap

    \n
  • HashMap is not preloaded, so it needs to be included use std::collections::HashMap
    1
    2
    3
    4
    5
    6
    7
    use std::collections::HashMap;
    fn main() {
        let keys = vec!["andy".to_string(), "cliff".to_string()] ;
        let ages = vec![38, 26];
        let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();
        println!("{:?}", map); /// print {"andy": 38, "cliff": 26}
    }
    • \n
\n

HashMap ownership

    \n
  • For types that implement the Copy trait (such as i32), the value will be copied into the HashMap
    • \n
  • For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap
    • \n
  • If a reference to a value is inserted into the HashMap, the value itself does not move
    • \n
\n

HashMap iteration

1
2
3
4
5
6
7
8
9
10
11
12
13
14
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);
    println!("{:?}", &map);
    for (k, v) in map {
        println!("{} age {}", k, v);
    } /// cliff age 26
      /// andy age 36
}
\n

update

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);

    let result = map.entry("bob".to_string());
    println!("{:?}", result); /// Entry(VacantEntry("bob"))

    let result = map.entry("andy".to_string());
    println!("{:?}", result); /// Entry(OccupiedEntry { key: "andy", value: 36, .. })

    map.entry("bob".to_string()).or_insert(28);
    map.entry("cliff".to_string()).or_insert(0);
}
\n\n

control flow

    \n
  • if
    1
    2
    3
    4
    5
    fn main() {
        let condition = 1;
        let x = if condition == 1 { "A" } else { "B" };
        println!("Result x = {}" , x) ;
    }
    • \n
  • loop
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    fn main() {
        let mut condition = 0;

        let result = 'outer: loop {  // 'outer is label
            'inner: loop {
                condition += 1;
                if 3 == condition {
                    break 'outer 3 * condition; // break outer loop
                }
            }
        };
        println!("Loop result is : {}", result); /// Loop result is : 9
    }

    • \n
  • rot
    1
    2
    3
    4
    5
    6
    fn main() {
        let arr = [3,2,3];
        for num in arr.iter() {
            println!("For value is {}", num);
        }
    }
    • \n
\n

Range iterator

    \n
  • Range
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
         for number in (1..=3) {
            println!("Number A is {}", number ); /// 1,2,3
        }
     
        for number in (1..=3).rev() { /// rev means reverse,
            println!("Number B is {}", number ); /// 3,2,1
        }
    }

    • \n
\n

struct

    \n
  • If struct is declared mutable then all fields in the instance are mutable
    • \n
\n

tuple struct

1
2
struct Color(i32,i32,i32);
let black = Color(0,0,0);
\n

Unit-Like struct

1
struct Man {};
\n

struct method

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

fn main() {
    let rec = Rectangle {
        width: 30,
        height: 50,
    };

    let result = rec.area(); 
    println!("rectangle：{:?}，area is：{}", rec, result);
}


#[derive(Debug)]
struct Rectangle {
    width: u32,
    height: u32,
}

impl Rectangle {
    fn area(&self) -> u32{
        self.width * self.height
    }
}

\n

associative func（similar to static method）

    \n
  • You can define a function that does not take self as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to String::from()
    1
    2
    3
    4
    5
    6
    7
    8
    impl Rectangle {
        fn create_square(width: u32) -> Rectangle {
            Rectangle {
                width,
                height: width,
            }
        }
    }
    • \n
\n

enum

1
2
3
4
5
6
7
8
9
enum Ip {
    V4,
    V6,
}

enum IpAddr {
    V4(String),
    V6(String),
}
\n\n

match

    \n
  • match must exhaust all possibilities
    • \n
  • If there are too many matchings, you can also use ““ for wildcarding, but note that ““ must be placed at the end
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    enum Color {
        Red,
        Yellow,
        Blue,
    }
    enum ColorWithVal {
        Red(u8,u8,u8),
        Yellow(u8,u8,u8),
        Blue(u8,u8,u8),
    }
    fn main(){
        let colour = Color::Blue;
        match colour {
            Color::Red => {
                println!("Red colour.");
            },
            _ => {
                println!("Other colour.");
            }
        }

        let colour = ColorWithVal::Red(222,111,22);
        match colour {
            ColorWithVal::Red(r,g,b) => {
                println!("Red colour. {},{},{}", r,g,b);
            },
            _ => {
                println!("Other colour.");
            }
        }
    }
    • \n
\n

if let

1
2
3
4
5
6
7
8
9
fn main(){
    let colour = Color::Red(Some(222),Some(222),Some(222));

    if let Color::Red(r,g,b) = colour {
        println!("Red colour. {:?},{:?},{:?}", r,g,b);
    } else {
        println!("Other colour.");
    }
}
\n\n

Result<T,E>

    \n
  • Recoverable err via Result<T,E>, non-recoverable via panic!
    • \n
  • upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program
    • \n
  • You can set panic = ‘abort’ in Cargo.toml to terminate the cleaning of the call stack
    1
    2
    [profile.release]
    panic='abort'
    • \n
  • RUST_BACKTRACE = 1 prints detailed error messages in the stack
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::fs::File;
    fn main() { 
        let fp = File::open("hello.txt");
        let file = match fp {
            Ok(file)=> {
                file
            },
            Err(error) => panic!("file not found {:?} ", error),
        };
    }
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
use std::{fs::File, io::ErrorKind};
fn main() { 
    let fp = File::open("hello.txt");
    let file = match fp {
        Ok(file)=> {
            file
        },
        Err(error) => {
            match error.kind() {
                ErrorKind::NotFound => {
                    match File::create("hello.txt") {
                        Ok(file) => {
                            file
                        },
                        Err(err) => {
                            panic!("file create error:{:?}", &err);
                        },
                    }
                },
                oe => panic!("other error {:?}", oe),
            }
        } ,
    };
}
\n
1
2
3
4
5
6
7
8
9
10
11
12
use std::{fs::File, io::ErrorKind};
fn main() { 
    let file = File::open("hello.txt").unwrap_or_else(|err| {
        if err.kind() == ErrorKind::NotFound {
            File::create("hello.txt").unwrap_or_else(|err|{
                panic!("error：{:?}", err);
            })
        }else{
            panic!("other error：{:?}", err);
        }
    });
}
\n

unwrap && expect

    \n
  • If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.
    • \n
  • expect can specify what the error message is, which is easier to debug
    • \n
\n

The question mark operator, ?

When writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.

\n
1
let mut file = File::create("my_best_friends.txt")?;
\n\n

generic

1
2
3
4
5
6
7
8
9
10
#[derive(Debug)]
struct Point<T, U>  {
   x : T,
   y : U,
}
impl <T, U> Point<T, U> {
    fn mixup<V, W>(self, other: Point<V, W>) -> Point<T, W> {
        Point{x: self.x , y: other.y, }
    }
}
\n\n

trait

definition

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
pub trait Summary {
    fn summarize(&self) -> String {
         "... more".to_string() /// default 
    }
}
pub struct Tweet {
    user_name :String,
    replay_count :u32,
    like_count :u32,
}
impl Tweet {
    fn like(&mut self) {
        self.like_count += 1;
    }
}

impl Summary for Tweet {
    fn summarize(&self) -> String {
        format!("{} like count :{} , replay count :{}", &self.user_name, &self.replay_count, &self.like_count)
    }
}
\n\n

trait as arguments

1
2
3
4
5
6
7
8
9
10
11
12
13
fn notify_msg <T:Summary> (info: T) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg (info: impl Summary + Display) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg <T> (info: T) 
where 
    T: Summary + Display,
{
    println!("summary : {}", info.summarize() );
    println!("display implement info : {}", info);
}
\n

trait as return

    \n
  • impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported
    • \n
\n

references

\n"},{"title":"rust ownership","date":"2022-10-11T09:09:28.000Z","_content":"\n## ownership rule\n- Each value in Rust has an owner.\n- There can only be one owner at a time.\n- When the owner goes out of scope, the value will be dropped.\n\n## variable scope\n```rust\nfn main() {\n { // s is not valid here, it’s not yet declared\n let s = \"hello\"; // s is valid from this point forward\n // do stuff with s\n } // this scope is now over, and s is no longer valid\n}\n```\n\n## Move\n### stack-only data: Copy trait\nsuch as primitive type\n```rust\nfn main() {\n let x = 5;\n let y = x;\n}\n```\nbind the value 5 to x; then make a copy of the value in x and bind it to y.\n\n### for heap variable\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1;\n}\n```\n![move-string](/images/rust/ownership/move-string.png)\nptr, len, capacity is stored in stack, while string value is stored in heap \\\nWhen we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap\n![move-string-2](/images/rust/ownership/move-string-2.png)\nsimilar to shallow copy\n\n## Clone\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1.clone();\n\n println!(\"s1 = {}, s2 = {}\", s1, s2);\n}\n```\nIf we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.\n\n### ownership and function\n- Passing a value to a function will result in a move or copy of ownership\n- difference between \"stack\" and \"heap\" variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable\n```rust\nfn main() {\n let s = String::from(\"hello\"); // s comes into scope\n takes_ownership(s); // s's value moves into the function...\n // ... and so is no longer valid here\n\n let x = 5; // x comes into scope\n\n makes_copy(x); // x would move into the function,\n // but i32 is Copy, so it's okay to still\n // use x afterward\n\n} // Here, x goes out of scope, then s. But because s's value was moved, nothing\n // special happens.\n\nfn takes_ownership(some_string: String) { // some_string comes into scope\n println!(\"{}\", some_string);\n} // Here, some_string goes out of scope and `drop` is called. The backing\n // memory is freed.\n\nfn makes_copy(some_integer: i32) { // some_integer comes into scope\n println!(\"{}\", some_integer);\n} // Here, some_integer goes out of scope. Nothing special happens.\n\n```\n\n### return values and scope\n```rust\nfn main() {\n let s1 = gives_ownership(); // gives_ownership moves its return\n // value into s1\n\n let s2 = String::from(\"hello\"); // s2 comes into scope\n\n let s3 = takes_and_gives_back(s2); // s2 is moved into\n // takes_and_gives_back, which also\n // moves its return value into s3\n} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing\n // happens. s1 goes out of scope and is dropped.\n\nfn gives_ownership() -> String { // gives_ownership will move its\n // return value into the function\n // that calls it\n\n let some_string = String::from(\"yours\"); // some_string comes into scope\n\n some_string // some_string is returned and\n // moves out to the calling\n // function\n}\n\n// This function takes a String and returns one\nfn takes_and_gives_back(a_string: String) -> String { // a_string comes into\n // scope\n\n a_string // a_string is returned and moves out to the calling function\n}\n```\n> What if we want to let a function use a value but not take ownership? \n> that's reference\n \n\n## reference & borrow\n- & means reference (borrow but not own)， default immutable\n- &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)\n- Multiple mutable references can be created non-simultaneously by creating a new scope\n- **Cannot have mutable and immutable references at the same time**\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n {\n let s1 = &mut s;\n } // r1 goes out of scope here, so we can make a new reference with no problems.\n let s2 = &mut s;\n}\n\nfn main() {\n let mut s = String::from(\"hello\");\n\n let r1 = &s; // no problem\n let r2 = &s; // no problem\n println!(\"{} and {}\", r1, r2);\n // variables r1 and r2 will not be used after this point\n\n let r3 = &mut s; // no problem\n println!(\"{}\", r3);\n\n println!{\"{}\",r1} // got problem with above mutable borrow\n}\n```\n\n### reference as function arguments\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n\n let len = calculate_length(&s1);\n\n println!(\"The length of '{}' is {}.\", s1, len);\n}\n\nfn calculate_length(s: &String) -> usize { // s is a reference to a String\n s.len() \n} // Here, s goes out of scope. But because it does not have ownership of what\n // it refers to, nothing happens.\n```\nwe pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used. \\\n When functions have references as parameters instead of the actual values, we won't need to return the values in order to give back ownership, because we never had ownership. \\\n We call the action of creating a reference borrowing. \n\n### mutable references\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n\n change(&mut s);\n}\n\nfn change(some_string: &mut String) {\n some_string.push_str(\", world\");\n}\n```\n\n### dangling references\n- A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else\n- rust，The compiler can guarantee that there will never be dangling references\n```rust\nfn main() {\n let r = dangle();\n}\nfn dangle() -> &string { // dangle returns a reference to a String\n let s = String::from(\"hello\"); // s is a new String\n &s // we return a reference to the String, s\n} // Here, s goes out of scope, and is dropped. Its memory goes away.\n// Danger\n```\nThe solution here is to return the String directly:\n```rust\nfn main() {\n let string = no_dangle();\n}\n\nfn no_dangle() -> String {\n let s = String::from(\"hello\");\n s\n}\n```\nThis works without any problems. Ownership is moved out, and nothing is deallocated.\n\n\n## slice\n- Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.\n\n### string slice\n```rust\nfn main() {\n let mut s = String::from(\"Hello world\");\n\n let hello = &s[0..5]; \n let world = &s[6..11];\n}\n```\nRather than a reference to the entire String, hello is a reference to a portion of the String, \\\nWith Rust's `..` range syntax, if you want to start at index zero, you can drop the value before the two periods \\\nBy the same token, if your slice includes the last byte of the String, you can drop the trailing number. \n> Note: String slice range indices must occur at **valid UTF-8 character boundaries**. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.\n\n```rust\nfn first_word(s :&String) -> &str {\n let bytes = s.as_bytes();\n for(i, &item) in bytes.iter().enumerate() {\n if item == b' ' {\n return &s[..i];\n }\n }\n &s[..]\n}\n```\n\n### String Literals Are Slices\n```rust\nfn main() {\nlet s = \"Hello, world!\";\n}\n```\nThe type of s here is &str: it's a slice pointing to that specific point of the binary. \n\n### String Slices as Parameters\n- Pass &str as a parameter, you can receive parameters of type &String and &str at the same time\n```rust\nfn first_word(s: &String) -> &str\n```\nequivalent to\n```rust\nfn first_word(s: &str) -> &str\n```\n\n### other slices\narray slice\n```rust\nfn main() {\n let a = [1, 2, 3, 4, 5];\n let slice = &a[1..3];\n assert_eq!(slice, &[2, 3]);\n}\n```","source":"_posts/rust/rust-03-ownership.md","raw":"---\ntitle: rust ownership\ndate: 2022-10-11 17:09:28\ntags: [rust]\n---\n\n## ownership rule\n- Each value in Rust has an owner.\n- There can only be one owner at a time.\n- When the owner goes out of scope, the value will be dropped.\n\n## variable scope\n```rust\nfn main() {\n { // s is not valid here, it’s not yet declared\n let s = \"hello\"; // s is valid from this point forward\n // do stuff with s\n } // this scope is now over, and s is no longer valid\n}\n```\n\n## Move\n### stack-only data: Copy trait\nsuch as primitive type\n```rust\nfn main() {\n let x = 5;\n let y = x;\n}\n```\nbind the value 5 to x; then make a copy of the value in x and bind it to y.\n\n### for heap variable\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1;\n}\n```\n![move-string](/images/rust/ownership/move-string.png)\nptr, len, capacity is stored in stack, while string value is stored in heap \\\nWhen we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap\n![move-string-2](/images/rust/ownership/move-string-2.png)\nsimilar to shallow copy\n\n## Clone\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1.clone();\n\n println!(\"s1 = {}, s2 = {}\", s1, s2);\n}\n```\nIf we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.\n\n### ownership and function\n- Passing a value to a function will result in a move or copy of ownership\n- difference between \"stack\" and \"heap\" variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable\n```rust\nfn main() {\n let s = String::from(\"hello\"); // s comes into scope\n takes_ownership(s); // s's value moves into the function...\n // ... and so is no longer valid here\n\n let x = 5; // x comes into scope\n\n makes_copy(x); // x would move into the function,\n // but i32 is Copy, so it's okay to still\n // use x afterward\n\n} // Here, x goes out of scope, then s. But because s's value was moved, nothing\n // special happens.\n\nfn takes_ownership(some_string: String) { // some_string comes into scope\n println!(\"{}\", some_string);\n} // Here, some_string goes out of scope and `drop` is called. The backing\n // memory is freed.\n\nfn makes_copy(some_integer: i32) { // some_integer comes into scope\n println!(\"{}\", some_integer);\n} // Here, some_integer goes out of scope. Nothing special happens.\n\n```\n\n### return values and scope\n```rust\nfn main() {\n let s1 = gives_ownership(); // gives_ownership moves its return\n // value into s1\n\n let s2 = String::from(\"hello\"); // s2 comes into scope\n\n let s3 = takes_and_gives_back(s2); // s2 is moved into\n // takes_and_gives_back, which also\n // moves its return value into s3\n} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing\n // happens. s1 goes out of scope and is dropped.\n\nfn gives_ownership() -> String { // gives_ownership will move its\n // return value into the function\n // that calls it\n\n let some_string = String::from(\"yours\"); // some_string comes into scope\n\n some_string // some_string is returned and\n // moves out to the calling\n // function\n}\n\n// This function takes a String and returns one\nfn takes_and_gives_back(a_string: String) -> String { // a_string comes into\n // scope\n\n a_string // a_string is returned and moves out to the calling function\n}\n```\n> What if we want to let a function use a value but not take ownership? \n> that's reference\n \n\n## reference & borrow\n- & means reference (borrow but not own)， default immutable\n- &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)\n- Multiple mutable references can be created non-simultaneously by creating a new scope\n- **Cannot have mutable and immutable references at the same time**\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n {\n let s1 = &mut s;\n } // r1 goes out of scope here, so we can make a new reference with no problems.\n let s2 = &mut s;\n}\n\nfn main() {\n let mut s = String::from(\"hello\");\n\n let r1 = &s; // no problem\n let r2 = &s; // no problem\n println!(\"{} and {}\", r1, r2);\n // variables r1 and r2 will not be used after this point\n\n let r3 = &mut s; // no problem\n println!(\"{}\", r3);\n\n println!{\"{}\",r1} // got problem with above mutable borrow\n}\n```\n\n### reference as function arguments\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n\n let len = calculate_length(&s1);\n\n println!(\"The length of '{}' is {}.\", s1, len);\n}\n\nfn calculate_length(s: &String) -> usize { // s is a reference to a String\n s.len() \n} // Here, s goes out of scope. But because it does not have ownership of what\n // it refers to, nothing happens.\n```\nwe pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used. \\\n When functions have references as parameters instead of the actual values, we won't need to return the values in order to give back ownership, because we never had ownership. \\\n We call the action of creating a reference borrowing. \n\n### mutable references\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n\n change(&mut s);\n}\n\nfn change(some_string: &mut String) {\n some_string.push_str(\", world\");\n}\n```\n\n### dangling references\n- A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else\n- rust，The compiler can guarantee that there will never be dangling references\n```rust\nfn main() {\n let r = dangle();\n}\nfn dangle() -> &string { // dangle returns a reference to a String\n let s = String::from(\"hello\"); // s is a new String\n &s // we return a reference to the String, s\n} // Here, s goes out of scope, and is dropped. Its memory goes away.\n// Danger\n```\nThe solution here is to return the String directly:\n```rust\nfn main() {\n let string = no_dangle();\n}\n\nfn no_dangle() -> String {\n let s = String::from(\"hello\");\n s\n}\n```\nThis works without any problems. Ownership is moved out, and nothing is deallocated.\n\n\n## slice\n- Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.\n\n### string slice\n```rust\nfn main() {\n let mut s = String::from(\"Hello world\");\n\n let hello = &s[0..5]; \n let world = &s[6..11];\n}\n```\nRather than a reference to the entire String, hello is a reference to a portion of the String, \\\nWith Rust's `..` range syntax, if you want to start at index zero, you can drop the value before the two periods \\\nBy the same token, if your slice includes the last byte of the String, you can drop the trailing number. \n> Note: String slice range indices must occur at **valid UTF-8 character boundaries**. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.\n\n```rust\nfn first_word(s :&String) -> &str {\n let bytes = s.as_bytes();\n for(i, &item) in bytes.iter().enumerate() {\n if item == b' ' {\n return &s[..i];\n }\n }\n &s[..]\n}\n```\n\n### String Literals Are Slices\n```rust\nfn main() {\nlet s = \"Hello, world!\";\n}\n```\nThe type of s here is &str: it's a slice pointing to that specific point of the binary. \n\n### String Slices as Parameters\n- Pass &str as a parameter, you can receive parameters of type &String and &str at the same time\n```rust\nfn first_word(s: &String) -> &str\n```\nequivalent to\n```rust\nfn first_word(s: &str) -> &str\n```\n\n### other slices\narray slice\n```rust\nfn main() {\n let a = [1, 2, 3, 4, 5];\n let slice = &a[1..3];\n assert_eq!(slice, &[2, 3]);\n}\n```","slug":"rust/rust-03-ownership","published":1,"updated":"2023-05-01T13:17:32.602Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzo000qofsjg48t6z8b","content":"

ownership rule

    \n
  • Each value in Rust has an owner.
    • \n
  • There can only be one owner at a time.
    • \n
  • When the owner goes out of scope, the value will be dropped.
    • \n
\n

variable scope

1
2
3
4
5
6
fn main() {
    {                      // s is not valid here, it’s not yet declared
        let s = "hello";   // s is valid from this point forward
        // do stuff with s
    }                      // this scope is now over, and s is no longer valid
}
\n\n

Move

stack-only data: Copy trait

such as primitive type

\n
1
2
3
4
fn main() {
    let x = 5;
    let y = x;
}
\n

bind the value 5 to x; then make a copy of the value in x and bind it to y.

\n

for heap variable

1
2
3
4
fn main() {
    let s1 = String::from("hello");
    let s2 = s1;
}
\n

\"move-string\"
ptr, len, capacity is stored in stack, while string value is stored in heap
When we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap
\"move-string-2\"
similar to shallow copy

\n

Clone

1
2
3
4
5
6
fn main() {
    let s1 = String::from("hello");
    let s2 = s1.clone();

    println!("s1 = {}, s2 = {}", s1, s2);
}
\n

If we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.

\n

ownership and function

    \n
  • Passing a value to a function will result in a move or copy of ownership
    • \n
  • difference between “stack” and “heap” variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    fn main() {
        let s = String::from("hello");  // s comes into scope
        takes_ownership(s);             // s's value moves into the function...
                                        // ... and so is no longer valid here

        let x = 5;                      // x comes into scope

        makes_copy(x);                  // x would move into the function,
                                        // but i32 is Copy, so it's okay to still
                                        // use x afterward

    } // Here, x goes out of scope, then s. But because s's value was moved, nothing
      // special happens.

    fn takes_ownership(some_string: String) { // some_string comes into scope
        println!("{}", some_string);
    } // Here, some_string goes out of scope and `drop` is called. The backing
      // memory is freed.

    fn makes_copy(some_integer: i32) { // some_integer comes into scope
        println!("{}", some_integer);
    } // Here, some_integer goes out of scope. Nothing special happens.

    • \n
\n

return values and scope

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
fn main() {
    let s1 = gives_ownership();         // gives_ownership moves its return
                                        // value into s1

    let s2 = String::from("hello");     // s2 comes into scope

    let s3 = takes_and_gives_back(s2);  // s2 is moved into
                                        // takes_and_gives_back, which also
                                        // moves its return value into s3
} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing
  // happens. s1 goes out of scope and is dropped.

fn gives_ownership() -> String {             // gives_ownership will move its
                                             // return value into the function
                                             // that calls it

    let some_string = String::from("yours"); // some_string comes into scope

    some_string                              // some_string is returned and
                                             // moves out to the calling
                                             // function
}

// This function takes a String and returns one
fn takes_and_gives_back(a_string: String) -> String { // a_string comes into
                                                      // scope

    a_string  // a_string is returned and moves out to the calling function
}
\n
\n

What if we want to let a function use a value but not take ownership?
that’s reference

\n
\n

reference & borrow

    \n
  • & means reference (borrow but not own)， default immutable
    • \n
  • &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)
    • \n
  • Multiple mutable references can be created non-simultaneously by creating a new scope
    • \n
  • Cannot have mutable and immutable references at the same time
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    fn main() {
        let mut s = String::from("hello");
        {
            let s1 = &mut s;
        } // r1 goes out of scope here, so we can make a new reference with no problems.
        let s2 = &mut s;
    }

    fn main() {
        let mut s = String::from("hello");

        let r1 = &s; // no problem
        let r2 = &s; // no problem
        println!("{} and {}", r1, r2);
        // variables r1 and r2 will not be used after this point

        let r3 = &mut s; // no problem
        println!("{}", r3);

        println!{"{}",r1} // got problem with above mutable borrow
    }
    • \n
\n

reference as function arguments

1
2
3
4
5
6
7
8
9
10
11
12
fn main() {
    let s1 = String::from("hello");

    let len = calculate_length(&s1);

    println!("The length of '{}' is {}.", s1, len);
}

fn calculate_length(s: &String) -> usize { // s is a reference to a String
    s.len() 
}  // Here, s goes out of scope. But because it does not have ownership of what
   // it refers to, nothing happens.
\n

we pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used.
When functions have references as parameters instead of the actual values, we won’t need to return the values in order to give back ownership, because we never had ownership.
We call the action of creating a reference borrowing.

\n

mutable references

1
2
3
4
5
6
7
8
9
fn main() {
    let mut s = String::from("hello");

    change(&mut s);
}

fn change(some_string: &mut String) {
    some_string.push_str(", world");
}
\n\n

dangling references

    \n
  • A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else
    • \n
  • rust，The compiler can guarantee that there will never be dangling references
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let r = dangle();
    }
    fn dangle() -> &string {  // dangle returns a reference to a String
        let s = String::from("hello");  // s is a new String
        &s // we return a reference to the String, s
    } // Here, s goes out of scope, and is dropped. Its memory goes away.
    // Danger
    \nThe solution here is to return the String directly:
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let string = no_dangle();
    }

    fn no_dangle() -> String {
        let s = String::from("hello");
        s
    }
    \nThis works without any problems. Ownership is moved out, and nothing is deallocated.
    • \n
\n

slice

    \n
  • Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.
    • \n
\n

string slice

1
2
3
4
5
6
fn main() {
    let mut s = String::from("Hello world");

    let hello = &s[0..5]; 
    let world = &s[6..11];
}
\n

Rather than a reference to the entire String, hello is a reference to a portion of the String,
With Rust’s .. range syntax, if you want to start at index zero, you can drop the value before the two periods
By the same token, if your slice includes the last byte of the String, you can drop the trailing number.

\n
\n

Note: String slice range indices must occur at valid UTF-8 character boundaries. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.

\n
\n
1
2
3
4
5
6
7
8
9
fn first_word(s :&String) -> &str {
    let bytes = s.as_bytes();
    for(i, &item) in bytes.iter().enumerate() {
        if item == b' ' {
            return &s[..i];
        }
    }
    &s[..]
}
\n\n

String Literals Are Slices

1
2
3
fn main() {
let s = "Hello, world!";
}
\n

The type of s here is &str: it’s a slice pointing to that specific point of the binary.

\n

String Slices as Parameters

    \n
  • Pass &str as a parameter, you can receive parameters of type &String and &str at the same time
    1
    fn first_word(s: &String) -> &str
    \nequivalent to
    1
    fn first_word(s: &str) -> &str
    • \n
\n

other slices

array slice

\n
1
2
3
4
5
fn main() {
  let a = [1, 2, 3, 4, 5];
  let slice = &a[1..3];
  assert_eq!(slice, &[2, 3]);
}
","site":{"data":{}},"excerpt":"","more":"

ownership rule

    \n
  • Each value in Rust has an owner.
    • \n
  • There can only be one owner at a time.
    • \n
  • When the owner goes out of scope, the value will be dropped.
    • \n
\n

variable scope

1
2
3
4
5
6
fn main() {
    {                      // s is not valid here, it’s not yet declared
        let s = "hello";   // s is valid from this point forward
        // do stuff with s
    }                      // this scope is now over, and s is no longer valid
}
\n\n

Move

stack-only data: Copy trait

such as primitive type

\n
1
2
3
4
fn main() {
    let x = 5;
    let y = x;
}
\n

bind the value 5 to x; then make a copy of the value in x and bind it to y.

\n

for heap variable

1
2
3
4
fn main() {
    let s1 = String::from("hello");
    let s2 = s1;
}
\n

\"move-string\"
ptr, len, capacity is stored in stack, while string value is stored in heap
When we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap
\"move-string-2\"
similar to shallow copy

\n

Clone

1
2
3
4
5
6
fn main() {
    let s1 = String::from("hello");
    let s2 = s1.clone();

    println!("s1 = {}, s2 = {}", s1, s2);
}
\n

If we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.

\n

ownership and function

    \n
  • Passing a value to a function will result in a move or copy of ownership
    • \n
  • difference between “stack” and “heap” variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    fn main() {
        let s = String::from("hello");  // s comes into scope
        takes_ownership(s);             // s's value moves into the function...
                                        // ... and so is no longer valid here

        let x = 5;                      // x comes into scope

        makes_copy(x);                  // x would move into the function,
                                        // but i32 is Copy, so it's okay to still
                                        // use x afterward

    } // Here, x goes out of scope, then s. But because s's value was moved, nothing
      // special happens.

    fn takes_ownership(some_string: String) { // some_string comes into scope
        println!("{}", some_string);
    } // Here, some_string goes out of scope and `drop` is called. The backing
      // memory is freed.

    fn makes_copy(some_integer: i32) { // some_integer comes into scope
        println!("{}", some_integer);
    } // Here, some_integer goes out of scope. Nothing special happens.

    • \n
\n

return values and scope

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
fn main() {
    let s1 = gives_ownership();         // gives_ownership moves its return
                                        // value into s1

    let s2 = String::from("hello");     // s2 comes into scope

    let s3 = takes_and_gives_back(s2);  // s2 is moved into
                                        // takes_and_gives_back, which also
                                        // moves its return value into s3
} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing
  // happens. s1 goes out of scope and is dropped.

fn gives_ownership() -> String {             // gives_ownership will move its
                                             // return value into the function
                                             // that calls it

    let some_string = String::from("yours"); // some_string comes into scope

    some_string                              // some_string is returned and
                                             // moves out to the calling
                                             // function
}

// This function takes a String and returns one
fn takes_and_gives_back(a_string: String) -> String { // a_string comes into
                                                      // scope

    a_string  // a_string is returned and moves out to the calling function
}
\n
\n

What if we want to let a function use a value but not take ownership?
that’s reference

\n
\n

reference & borrow

    \n
  • & means reference (borrow but not own)， default immutable
    • \n
  • &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)
    • \n
  • Multiple mutable references can be created non-simultaneously by creating a new scope
    • \n
  • Cannot have mutable and immutable references at the same time
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    fn main() {
        let mut s = String::from("hello");
        {
            let s1 = &mut s;
        } // r1 goes out of scope here, so we can make a new reference with no problems.
        let s2 = &mut s;
    }

    fn main() {
        let mut s = String::from("hello");

        let r1 = &s; // no problem
        let r2 = &s; // no problem
        println!("{} and {}", r1, r2);
        // variables r1 and r2 will not be used after this point

        let r3 = &mut s; // no problem
        println!("{}", r3);

        println!{"{}",r1} // got problem with above mutable borrow
    }
    • \n
\n

reference as function arguments

1
2
3
4
5
6
7
8
9
10
11
12
fn main() {
    let s1 = String::from("hello");

    let len = calculate_length(&s1);

    println!("The length of '{}' is {}.", s1, len);
}

fn calculate_length(s: &String) -> usize { // s is a reference to a String
    s.len() 
}  // Here, s goes out of scope. But because it does not have ownership of what
   // it refers to, nothing happens.
\n

we pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used.
When functions have references as parameters instead of the actual values, we won’t need to return the values in order to give back ownership, because we never had ownership.
We call the action of creating a reference borrowing.

\n

mutable references

1
2
3
4
5
6
7
8
9
fn main() {
    let mut s = String::from("hello");

    change(&mut s);
}

fn change(some_string: &mut String) {
    some_string.push_str(", world");
}
\n\n

dangling references

    \n
  • A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else
    • \n
  • rust，The compiler can guarantee that there will never be dangling references
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let r = dangle();
    }
    fn dangle() -> &string {  // dangle returns a reference to a String
        let s = String::from("hello");  // s is a new String
        &s // we return a reference to the String, s
    } // Here, s goes out of scope, and is dropped. Its memory goes away.
    // Danger
    \nThe solution here is to return the String directly:
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let string = no_dangle();
    }

    fn no_dangle() -> String {
        let s = String::from("hello");
        s
    }
    \nThis works without any problems. Ownership is moved out, and nothing is deallocated.
    • \n
\n

slice

    \n
  • Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.
    • \n
\n

string slice

1
2
3
4
5
6
fn main() {
    let mut s = String::from("Hello world");

    let hello = &s[0..5]; 
    let world = &s[6..11];
}
\n

Rather than a reference to the entire String, hello is a reference to a portion of the String,
With Rust’s .. range syntax, if you want to start at index zero, you can drop the value before the two periods
By the same token, if your slice includes the last byte of the String, you can drop the trailing number.

\n
\n

Note: String slice range indices must occur at valid UTF-8 character boundaries. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.

\n
\n
1
2
3
4
5
6
7
8
9
fn first_word(s :&String) -> &str {
    let bytes = s.as_bytes();
    for(i, &item) in bytes.iter().enumerate() {
        if item == b' ' {
            return &s[..i];
        }
    }
    &s[..]
}
\n\n

String Literals Are Slices

1
2
3
fn main() {
let s = "Hello, world!";
}
\n

The type of s here is &str: it’s a slice pointing to that specific point of the binary.

\n

String Slices as Parameters

    \n
  • Pass &str as a parameter, you can receive parameters of type &String and &str at the same time
    1
    fn first_word(s: &String) -> &str
    \nequivalent to
    1
    fn first_word(s: &str) -> &str
    • \n
\n

other slices

array slice

\n
1
2
3
4
5
fn main() {
  let a = [1, 2, 3, 4, 5];
  let slice = &a[1..3];
  assert_eq!(slice, &[2, 3]);
}
"},{"title":"rust lifetime","date":"2022-10-18T13:33:26.000Z","_content":"\n## lifetime\n- Every reference in Rust has its own lifecycle\n- most of the time, Rust's lifetime is implicit and can be inferred\n- There are two types of life cycle: input life cycle and output life cycle\n- 'static is a special life cycle annotation\n### Example of lifetime out of scope\n```rust\nfn main() {\n let mut x;\n {\n let y = String::from(\"hello\");\n // x = y; // this is allowed\n x = &y; // not allowed. borrowed value (y) does not live long enough\n }\n println!(\"Str:{}\", x);\n}\n```\n\n### lifetime checker\nRust compiler's borrow checker to determine whether a borrow is legal\n```rust\n// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`\nfn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\nlet's find out why it is such case\n```rust\nfn main() {\n // variable to hold the result value\n let long_str; \n\n let x = \"abc\".to_string();\n {\n let y = \"bbccd\".to_string();\n long_str = longest(x.as_str(), y.as_str());\n }\n // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value\n println!(\"Longest str: {}\", long_str);\n}\n\n```\nHence, we need lifetime annotation `'`\n```rust\nfn longest<'a>(x:&'a str, y:&'a str) -> &'a str {\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\n\n### deeper understanding\n- When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters\n\n### Struct lifetime annotation\n```rust\nfn main() {\n let info = String::from(\"File not found.\");\n // 存放结果值的变量\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton<'a> {\n part: &'a str,\n}\n```\n- lifetime of the field `part` must be longer than struct\n\n### Lifetime Elision\nIn order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.\nElision rules are as follows:\n- Each elided lifetime in input position becomes a distinct lifetime parameter.\n- If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.\n- If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.\n- Otherwise, it is an error to elide an output lifetime.\n\n### struct lifetime annotation\n```rust\n\nfn main() {\n let info = String::from(\"File not found.\");\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton <'a>{\n part: &'a str,\n}\n\n// the first 'a is decraration, the second is usage\nimpl<'a> ImportantExcepiton <'a> {\n // in return value, 'a is omitted according to Lifetime Elision rule\n fn callname(&self ) -> &str{\n self.part\n }\n}\n```\n\n### 'static\n'static is a special lifetime that takes up the duration of the entire program, for example all string literals have a 'static lifetime","source":"_posts/rust/rust-04-lifetime.md","raw":"---\ntitle: rust lifetime\ndate: 2022-10-18 21:33:26\ntags: [rust]\n---\n\n## lifetime\n- Every reference in Rust has its own lifecycle\n- most of the time, Rust's lifetime is implicit and can be inferred\n- There are two types of life cycle: input life cycle and output life cycle\n- 'static is a special life cycle annotation\n### Example of lifetime out of scope\n```rust\nfn main() {\n let mut x;\n {\n let y = String::from(\"hello\");\n // x = y; // this is allowed\n x = &y; // not allowed. borrowed value (y) does not live long enough\n }\n println!(\"Str:{}\", x);\n}\n```\n\n### lifetime checker\nRust compiler's borrow checker to determine whether a borrow is legal\n```rust\n// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`\nfn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\nlet's find out why it is such case\n```rust\nfn main() {\n // variable to hold the result value\n let long_str; \n\n let x = \"abc\".to_string();\n {\n let y = \"bbccd\".to_string();\n long_str = longest(x.as_str(), y.as_str());\n }\n // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value\n println!(\"Longest str: {}\", long_str);\n}\n\n```\nHence, we need lifetime annotation `'`\n```rust\nfn longest<'a>(x:&'a str, y:&'a str) -> &'a str {\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\n\n### deeper understanding\n- When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters\n\n### Struct lifetime annotation\n```rust\nfn main() {\n let info = String::from(\"File not found.\");\n // 存放结果值的变量\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton<'a> {\n part: &'a str,\n}\n```\n- lifetime of the field `part` must be longer than struct\n\n### Lifetime Elision\nIn order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.\nElision rules are as follows:\n- Each elided lifetime in input position becomes a distinct lifetime parameter.\n- If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.\n- If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.\n- Otherwise, it is an error to elide an output lifetime.\n\n### struct lifetime annotation\n```rust\n\nfn main() {\n let info = String::from(\"File not found.\");\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton <'a>{\n part: &'a str,\n}\n\n// the first 'a is decraration, the second is usage\nimpl<'a> ImportantExcepiton <'a> {\n // in return value, 'a is omitted according to Lifetime Elision rule\n fn callname(&self ) -> &str{\n self.part\n }\n}\n```\n\n### 'static\n'static is a special lifetime that takes up the duration of the entire program, for example all string literals have a 'static lifetime","slug":"rust/rust-04-lifetime","published":1,"updated":"2023-05-01T14:08:49.387Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzp000sofsj8j5g4s8r","content":"

lifetime

    \n
  • Every reference in Rust has its own lifecycle
    • \n
  • most of the time, Rust’s lifetime is implicit and can be inferred
    • \n
  • There are two types of life cycle: input life cycle and output life cycle
    • \n
  • ‘static is a special life cycle annotation
    • \n
\n

Example of lifetime out of scope

1
2
3
4
5
6
7
8
9
fn main() {
    let mut x;
    {
        let y = String::from("hello");
        // x = y; // this is allowed
        x = &y; // not allowed. borrowed value (y) does not live long enough
    }
    println!("Str:{}", x);
}
\n\n

lifetime checker

Rust compiler’s borrow checker to determine whether a borrow is legal

\n
1
2
3
4
5
6
7
8
// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`
fn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n

let’s find out why it is such case

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    // variable to hold the result value
    let long_str; 

    let x = "abc".to_string();
    {
        let y = "bbccd".to_string();
        long_str = longest(x.as_str(), y.as_str());
    }
    // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value
    println!("Longest str: {}", long_str);
}

\n

Hence, we need lifetime annotation '

\n
1
2
3
4
5
6
7
fn longest<'a>(x:&'a str, y:&'a str) -> &'a str {
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n\n

deeper understanding

    \n
  • When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters
    • \n
\n

Struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    let info = String::from("File not found.");
    // 存放结果值的变量
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton<'a> {
    part: &'a str,
}
\n
    \n
  • lifetime of the field part must be longer than struct
    • \n
\n

Lifetime Elision

In order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.
Elision rules are as follows:

\n
    \n
  • Each elided lifetime in input position becomes a distinct lifetime parameter.
    • \n
  • If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.
    • \n
  • If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.
    • \n
  • Otherwise, it is an error to elide an output lifetime.
    • \n
\n

struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

fn main() {
    let info = String::from("File not found.");
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton <'a>{
    part: &'a str,
}

// the first 'a is decraration, the second is usage
impl<'a> ImportantExcepiton <'a> {
    // in return value, 'a is omitted according to Lifetime Elision rule
    fn callname(&self ) -> &str{
        self.part
    }
}
\n\n

‘static

‘static is a special lifetime that takes up the duration of the entire program, for example all string literals have a ‘static lifetime

\n","site":{"data":{}},"excerpt":"","more":"

lifetime

    \n
  • Every reference in Rust has its own lifecycle
    • \n
  • most of the time, Rust’s lifetime is implicit and can be inferred
    • \n
  • There are two types of life cycle: input life cycle and output life cycle
    • \n
  • ‘static is a special life cycle annotation
    • \n
\n

Example of lifetime out of scope

1
2
3
4
5
6
7
8
9
fn main() {
    let mut x;
    {
        let y = String::from("hello");
        // x = y; // this is allowed
        x = &y; // not allowed. borrowed value (y) does not live long enough
    }
    println!("Str:{}", x);
}
\n\n

lifetime checker

Rust compiler’s borrow checker to determine whether a borrow is legal

\n
1
2
3
4
5
6
7
8
// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`
fn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n

let’s find out why it is such case

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    // variable to hold the result value
    let long_str; 

    let x = "abc".to_string();
    {
        let y = "bbccd".to_string();
        long_str = longest(x.as_str(), y.as_str());
    }
    // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value
    println!("Longest str: {}", long_str);
}

\n

Hence, we need lifetime annotation '

\n
1
2
3
4
5
6
7
fn longest<'a>(x:&'a str, y:&'a str) -> &'a str {
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n\n

deeper understanding

    \n
  • When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters
    • \n
\n

Struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    let info = String::from("File not found.");
    // 存放结果值的变量
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton<'a> {
    part: &'a str,
}
\n
    \n
  • lifetime of the field part must be longer than struct
    • \n
\n

Lifetime Elision

In order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.
Elision rules are as follows:

\n
    \n
  • Each elided lifetime in input position becomes a distinct lifetime parameter.
    • \n
  • If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.
    • \n
  • If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.
    • \n
  • Otherwise, it is an error to elide an output lifetime.
    • \n
\n

struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

fn main() {
    let info = String::from("File not found.");
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton <'a>{
    part: &'a str,
}

// the first 'a is decraration, the second is usage
impl<'a> ImportantExcepiton <'a> {
    // in return value, 'a is omitted according to Lifetime Elision rule
    fn callname(&self ) -> &str{
        self.part
    }
}
\n\n

‘static

‘static is a special lifetime that takes up the duration of the entire program, for example all string literals have a ‘static lifetime

\n"},{"title":"rust memory layout","date":"2022-10-25T02:54:13.000Z","_content":"\n\n## trait object\na reference to a trait type (or Box) is called trait object\n\n### two ways convert concrete type to trait object\n1. assigning to variable\n```rust\nuse std::io::Write;\n\nlet mut buffer: Vec = vec![];\nlet w: &mut dyn Write = &mut buffer;\n```\n2. pass a concrete type as a argument to a function\n```rust\nfn main() {\n let mut buffer: Vec = vec![];\n writer(&mut buffer);\n}\n\nfn writer(w: &mut dyn Write) {\n // ...\n}\n```\nin both ways, buffer is converted to a trait object implements Write\n\nin memory, a trait object (in the example, w) is a fat point consists two pointers\n![trait_object_memory](/images/rust/memory/trait_object_memory.png)\nthe vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a \"Writer\"\n\n\n## references\n- https://www.youtube.com/watch?v=rDoqT-a6UFg","source":"_posts/rust/rust-05-memory-layout.md","raw":"---\ntitle: rust memory layout\ndate: 2022-10-25 10:54:13\ntags: [rust]\n---\n\n\n## trait object\na reference to a trait type (or Box) is called trait object\n\n### two ways convert concrete type to trait object\n1. assigning to variable\n```rust\nuse std::io::Write;\n\nlet mut buffer: Vec = vec![];\nlet w: &mut dyn Write = &mut buffer;\n```\n2. pass a concrete type as a argument to a function\n```rust\nfn main() {\n let mut buffer: Vec = vec![];\n writer(&mut buffer);\n}\n\nfn writer(w: &mut dyn Write) {\n // ...\n}\n```\nin both ways, buffer is converted to a trait object implements Write\n\nin memory, a trait object (in the example, w) is a fat point consists two pointers\n![trait_object_memory](/images/rust/memory/trait_object_memory.png)\nthe vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a \"Writer\"\n\n\n## references\n- https://www.youtube.com/watch?v=rDoqT-a6UFg","slug":"rust/rust-05-memory-layout","published":1,"updated":"2023-05-03T02:57:52.719Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzp000uofsjgftu030k","content":"

trait object

a reference to a trait type (or Box) is called trait object

\n

two ways convert concrete type to trait object

    \n
  1. assigning to variable
    1
    2
    3
    4
    use std::io::Write;

    let mut buffer: Vec<u8> = vec![];
    let w: &mut dyn Write = &mut buffer;
    2. \n
  2. pass a concrete type as a argument to a function
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
       let mut buffer: Vec<u8> = vec![];
       writer(&mut buffer);
    }

    fn writer(w: &mut dyn Write) {
        // ...
    }
    \nin both ways, buffer is converted to a trait object implements Write
    3. \n
\n

in memory, a trait object (in the example, w) is a fat point consists two pointers
\"trait_object_memory\"
the vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a “Writer”

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

trait object

a reference to a trait type (or Box) is called trait object

\n

two ways convert concrete type to trait object

    \n
  1. assigning to variable
    1
    2
    3
    4
    use std::io::Write;

    let mut buffer: Vec<u8> = vec![];
    let w: &mut dyn Write = &mut buffer;
    2. \n
  2. pass a concrete type as a argument to a function
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
       let mut buffer: Vec<u8> = vec![];
       writer(&mut buffer);
    }

    fn writer(w: &mut dyn Write) {
        // ...
    }
    \nin both ways, buffer is converted to a trait object implements Write
    3. \n
\n

in memory, a trait object (in the example, w) is a fat point consists two pointers
\"trait_object_memory\"
the vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a “Writer”

\n

references

\n"},{"title":"rust reflect and macro","date":"2022-12-28T02:24:21.000Z","_content":"\n## reflect\n### trait object\nRust provides dynamic dispatch through a feature called `trait objects`. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at **runtime**. more details can be found on [references 1]\n\n### any\nThis module (std::any) contains the `Any` trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the `Provider` trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.\nAny itself can be used to get a TypeId\n\n```rust\nuse std::fmt::Debug;\nuse std::any::Any;\n\nfn log(value: &Any) {\n let value_any = value as &dyn Any; // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.\n\n match value_any.downcast_ref::() {\n Some(val_str) -> {\n // do with string\n },\n None => {\n // \n }\n }\n}\n```\n\n### porpular crates using Any\n- [oso](https://docs.osohq.com/)\nThe Oso Library is a batteries-included framework for building authorization in your application\n- [bevy](https://bevyengine.org/)\nA refreshingly simple data-driven game engine built in Rust\n\n## macro\n### rust compile process\n![rust compile process](/images/rust/macros/16.compile_process.png)\n\n### front end: rustc\n1. lexical analysis: Code Text -> TokenStream\n2. syntax analysis: TokenStream -> AST (abstract syntax tree)\n3. semantic analyzer: \n AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g `for` changed to `loop`) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)\n\n### back end: LLVM\nLLVM IR -> machine code\n\n\n### macros in compiling\n- declarative macros: TokenStream - expand -> TokenStream\n- procedule macros: self defined AST with the help or third party crate such as syn, quote\n\n## declarative macro: macro_rules!\nDeclarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed\n```rust\n#![allow(unused)]\nfn main() {\n match target {\n match_pattern_1 => expr_1,\n match_pattern_2 => {\n statement1;\n statement2;\n expr_2\n },\n _ => expr_3\n }\n}\n```\n\n### example 1, simplified `vec!`\nbelow example use macro_rules to implement a simplified version of vec!\n```rust\n#[macro_export]\nmacro_rules! vec {\n ( $( $x:expr ),* ) => {\n {\n let mut temp_vec = Vec::new();\n $(\n temp_vec.push($x);\n )*\n temp_vec\n }\n };\n}\n\n#![allow(unused)]\nfn main() {\n let v: Vec = vec![1, 2, 3];\n}\n```\n\n### example 2, unless\n```rust\nmacro_rules! unless {\n ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);\n}\n\nfn cmp(a:i32, b:i32) {\n unless!{\n (a>b) => {println!(\"{} < {}\", a,b);}\n }\n}\nfn main() {\n cmp(1,2); /// print \"1<2\" as the condition is true !(a>b)\n cmp(3,2); /// print nothing\n}\n```\n### example 3, HashMap\n```rust\nmacro_rules! hashmap {\n // match for \"a\" => 1, \"b\" => 2,\n ( $($key:expr => $value:expr,)* ) =>\n { hashmap!($($key => $value),*) }; // recuisive\n // match for \"a\" => 1, \"b\" => 2\n ( $($key:expr => $value:expr),* ) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\n\nmacro_rules! hashmap_equivalent {\n ( $($key:expr => $value:expr),* $(,)*) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\nfn main() {\n let map = hashmap!{\n \"a\" => 1,\n \"b\" => 2, // with or without ,\n };\n let map_2 = hashmap_equivalent!{\n \"a\" => 1, \n \"b\" => 2, // with or without ,\n };\n}\n```\n\n### metavariables\n- item: an Item\n- stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)\n- expr: an Expression\n- ty: a Type\n- ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER\n- path: a TypePath style path\n- tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})\n- meta: an Attr, the contents of an attribute\n- lifetime: a LIFETIME_TOKEN\n- vis: a possibly empty Visibility qualifier\n- literal: matches LiteralExpression\ndetails to be found [here](https://doc.rust-lang.org/reference/macros-by-example.html)\n\n## procedures macro\n\n## references\n1. [rust trait object](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/trait-objects.html)","source":"_posts/rust/rust-07-macro.md","raw":"---\ntitle: rust reflect and macro\ndate: 2022-12-28 10:24:21\ntags: [rust]\n---\n\n## reflect\n### trait object\nRust provides dynamic dispatch through a feature called `trait objects`. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at **runtime**. more details can be found on [references 1]\n\n### any\nThis module (std::any) contains the `Any` trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the `Provider` trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.\nAny itself can be used to get a TypeId\n\n```rust\nuse std::fmt::Debug;\nuse std::any::Any;\n\nfn log(value: &Any) {\n let value_any = value as &dyn Any; // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.\n\n match value_any.downcast_ref::() {\n Some(val_str) -> {\n // do with string\n },\n None => {\n // \n }\n }\n}\n```\n\n### porpular crates using Any\n- [oso](https://docs.osohq.com/)\nThe Oso Library is a batteries-included framework for building authorization in your application\n- [bevy](https://bevyengine.org/)\nA refreshingly simple data-driven game engine built in Rust\n\n## macro\n### rust compile process\n![rust compile process](/images/rust/macros/16.compile_process.png)\n\n### front end: rustc\n1. lexical analysis: Code Text -> TokenStream\n2. syntax analysis: TokenStream -> AST (abstract syntax tree)\n3. semantic analyzer: \n AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g `for` changed to `loop`) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)\n\n### back end: LLVM\nLLVM IR -> machine code\n\n\n### macros in compiling\n- declarative macros: TokenStream - expand -> TokenStream\n- procedule macros: self defined AST with the help or third party crate such as syn, quote\n\n## declarative macro: macro_rules!\nDeclarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed\n```rust\n#![allow(unused)]\nfn main() {\n match target {\n match_pattern_1 => expr_1,\n match_pattern_2 => {\n statement1;\n statement2;\n expr_2\n },\n _ => expr_3\n }\n}\n```\n\n### example 1, simplified `vec!`\nbelow example use macro_rules to implement a simplified version of vec!\n```rust\n#[macro_export]\nmacro_rules! vec {\n ( $( $x:expr ),* ) => {\n {\n let mut temp_vec = Vec::new();\n $(\n temp_vec.push($x);\n )*\n temp_vec\n }\n };\n}\n\n#![allow(unused)]\nfn main() {\n let v: Vec = vec![1, 2, 3];\n}\n```\n\n### example 2, unless\n```rust\nmacro_rules! unless {\n ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);\n}\n\nfn cmp(a:i32, b:i32) {\n unless!{\n (a>b) => {println!(\"{} < {}\", a,b);}\n }\n}\nfn main() {\n cmp(1,2); /// print \"1<2\" as the condition is true !(a>b)\n cmp(3,2); /// print nothing\n}\n```\n### example 3, HashMap\n```rust\nmacro_rules! hashmap {\n // match for \"a\" => 1, \"b\" => 2,\n ( $($key:expr => $value:expr,)* ) =>\n { hashmap!($($key => $value),*) }; // recuisive\n // match for \"a\" => 1, \"b\" => 2\n ( $($key:expr => $value:expr),* ) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\n\nmacro_rules! hashmap_equivalent {\n ( $($key:expr => $value:expr),* $(,)*) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\nfn main() {\n let map = hashmap!{\n \"a\" => 1,\n \"b\" => 2, // with or without ,\n };\n let map_2 = hashmap_equivalent!{\n \"a\" => 1, \n \"b\" => 2, // with or without ,\n };\n}\n```\n\n### metavariables\n- item: an Item\n- stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)\n- expr: an Expression\n- ty: a Type\n- ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER\n- path: a TypePath style path\n- tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})\n- meta: an Attr, the contents of an attribute\n- lifetime: a LIFETIME_TOKEN\n- vis: a possibly empty Visibility qualifier\n- literal: matches LiteralExpression\ndetails to be found [here](https://doc.rust-lang.org/reference/macros-by-example.html)\n\n## procedures macro\n\n## references\n1. [rust trait object](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/trait-objects.html)","slug":"rust/rust-07-macro","published":1,"updated":"2023-05-01T14:18:30.350Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzq000wofsj7u5g7rf7","content":"

reflect

trait object

Rust provides dynamic dispatch through a feature called trait objects. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at runtime. more details can be found on [references 1]

\n

any

This module (std::any) contains the Any trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the Provider trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.
Any itself can be used to get a TypeId

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
use std::fmt::Debug;
use std::any::Any;

fn log<T: Any + Debug>(value: &Any) {
    let value_any = value as &dyn Any;  // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.

    match value_any.downcast_ref::<String>() {
        Some(val_str) -> {
            // do with string
        },
        None => {
            // 
        }
    }
}
\n\n

porpular crates using Any

    \n
  • oso
    The Oso Library is a batteries-included framework for building authorization in your application
    • \n
  • bevy
    A refreshingly simple data-driven game engine built in Rust
    • \n
\n

macro

rust compile process

\"rust

\n

front end: rustc

    \n
  1. lexical analysis: Code Text -> TokenStream
    2. \n
  2. syntax analysis: TokenStream -> AST (abstract syntax tree)
    3. \n
  3. semantic analyzer:
    AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g for changed to loop) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)
    4. \n
\n

back end: LLVM

LLVM IR -> machine code

\n

macros in compiling

    \n
  • declarative macros: TokenStream - expand -> TokenStream
    • \n
  • procedule macros: self defined AST with the help or third party crate such as syn, quote
    • \n
\n

declarative macro: macro_rules!

Declarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed

\n
1
2
3
4
5
6
7
8
9
10
11
12
#![allow(unused)]
fn main() {
    match target {
        match_pattern_1 => expr_1,
        match_pattern_2 => {
            statement1;
            statement2;
            expr_2
        },
        _ => expr_3
    }
}
\n\n

example 1, simplified vec!

below example use macro_rules to implement a simplified version of vec!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#[macro_export]
macro_rules! vec {
    ( $( $x:expr ),* ) => {
        {
            let mut temp_vec = Vec::new();
            $(
                temp_vec.push($x);
            )*
            temp_vec
        }
    };
}

#![allow(unused)]
fn main() {
    let v: Vec<u32> = vec![1, 2, 3];
}
\n\n

example 2, unless

1
2
3
4
5
6
7
8
9
10
11
12
13
macro_rules! unless {
    ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);
}

fn cmp(a:i32, b:i32) {
    unless!{
        (a>b) => {println!("{} < {}", a,b);}
    }
}
fn main() {
    cmp(1,2);  /// print "1<2" as the condition is true !(a>b)
    cmp(3,2);  /// print nothing
}
\n

example 3, HashMap

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
macro_rules! hashmap {
    // match for "a" => 1, "b" => 2,
    ( $($key:expr => $value:expr,)* ) =>
        { hashmap!($($key => $value),*) }; // recuisive
    // match for "a" => 1, "b" => 2
    ( $($key:expr => $value:expr),* ) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}

macro_rules! hashmap_equivalent {
    ( $($key:expr => $value:expr),* $(,)*) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}
fn main() {
    let map = hashmap!{
        "a" => 1,
        "b" => 2,  // with or without ,
    };
    let map_2 = hashmap_equivalent!{
        "a" => 1, 
        "b" => 2,  // with or without ,
    };
}
\n\n

metavariables

    \n
  • item: an Item
    • \n
  • stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)
    • \n
  • expr: an Expression
    • \n
  • ty: a Type
    • \n
  • ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER
    • \n
  • path: a TypePath style path
    • \n
  • tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})
    • \n
  • meta: an Attr, the contents of an attribute
    • \n
  • lifetime: a LIFETIME_TOKEN
    • \n
  • vis: a possibly empty Visibility qualifier
    • \n
  • literal: matches LiteralExpression
    details to be found here
    • \n
\n

procedures macro

references

    \n
  1. rust trait object
    2. \n
\n","site":{"data":{}},"excerpt":"","more":"

reflect

trait object

Rust provides dynamic dispatch through a feature called trait objects. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at runtime. more details can be found on [references 1]

\n

any

This module (std::any) contains the Any trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the Provider trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.
Any itself can be used to get a TypeId

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
use std::fmt::Debug;
use std::any::Any;

fn log<T: Any + Debug>(value: &Any) {
    let value_any = value as &dyn Any;  // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.

    match value_any.downcast_ref::<String>() {
        Some(val_str) -> {
            // do with string
        },
        None => {
            // 
        }
    }
}
\n\n

porpular crates using Any

    \n
  • oso
    The Oso Library is a batteries-included framework for building authorization in your application
    • \n
  • bevy
    A refreshingly simple data-driven game engine built in Rust
    • \n
\n

macro

rust compile process

\"rust

\n

front end: rustc

    \n
  1. lexical analysis: Code Text -> TokenStream
    2. \n
  2. syntax analysis: TokenStream -> AST (abstract syntax tree)
    3. \n
  3. semantic analyzer:
    AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g for changed to loop) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)
    4. \n
\n

back end: LLVM

LLVM IR -> machine code

\n

macros in compiling

    \n
  • declarative macros: TokenStream - expand -> TokenStream
    • \n
  • procedule macros: self defined AST with the help or third party crate such as syn, quote
    • \n
\n

declarative macro: macro_rules!

Declarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed

\n
1
2
3
4
5
6
7
8
9
10
11
12
#![allow(unused)]
fn main() {
    match target {
        match_pattern_1 => expr_1,
        match_pattern_2 => {
            statement1;
            statement2;
            expr_2
        },
        _ => expr_3
    }
}
\n\n

example 1, simplified vec!

below example use macro_rules to implement a simplified version of vec!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#[macro_export]
macro_rules! vec {
    ( $( $x:expr ),* ) => {
        {
            let mut temp_vec = Vec::new();
            $(
                temp_vec.push($x);
            )*
            temp_vec
        }
    };
}

#![allow(unused)]
fn main() {
    let v: Vec<u32> = vec![1, 2, 3];
}
\n\n

example 2, unless

1
2
3
4
5
6
7
8
9
10
11
12
13
macro_rules! unless {
    ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);
}

fn cmp(a:i32, b:i32) {
    unless!{
        (a>b) => {println!("{} < {}", a,b);}
    }
}
fn main() {
    cmp(1,2);  /// print "1<2" as the condition is true !(a>b)
    cmp(3,2);  /// print nothing
}
\n

example 3, HashMap

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
macro_rules! hashmap {
    // match for "a" => 1, "b" => 2,
    ( $($key:expr => $value:expr,)* ) =>
        { hashmap!($($key => $value),*) }; // recuisive
    // match for "a" => 1, "b" => 2
    ( $($key:expr => $value:expr),* ) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}

macro_rules! hashmap_equivalent {
    ( $($key:expr => $value:expr),* $(,)*) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}
fn main() {
    let map = hashmap!{
        "a" => 1,
        "b" => 2,  // with or without ,
    };
    let map_2 = hashmap_equivalent!{
        "a" => 1, 
        "b" => 2,  // with or without ,
    };
}
\n\n

metavariables

    \n
  • item: an Item
    • \n
  • stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)
    • \n
  • expr: an Expression
    • \n
  • ty: a Type
    • \n
  • ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER
    • \n
  • path: a TypePath style path
    • \n
  • tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})
    • \n
  • meta: an Attr, the contents of an attribute
    • \n
  • lifetime: a LIFETIME_TOKEN
    • \n
  • vis: a possibly empty Visibility qualifier
    • \n
  • literal: matches LiteralExpression
    details to be found here
    • \n
\n

procedures macro

references

    \n
  1. rust trait object
    2. \n
\n"},{"title":"rust project management","date":"2022-11-06T07:33:55.000Z","_content":"\n## basic concepts\n- **Packages**: A Cargo feature that lets you build, test, and share crates\n- **Crates**: A tree of modules that produces a library or executable\n- **Modules and use**: Let you control the organization, scope, and privacy of paths\n- **Paths**: A way of naming an item, such as a struct, function, or module\n\n### package\n1. one package can only has one library Crate. default is src/lib.rs\n2. one package can have multiple binary Crates (under src/bin). default src/main.rs\n3. one package has at least one crate，no matter lib or bin。\n4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)\n```\ncargo new my-project --lib ## create a library project\n```\n### crate\n- binary or library\n- The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate\n\n### module\n- keyword `mod` \n- module could be nested。\n- module includes(struct、enum、const、trait、func, etc)\n\n### path\n- absolute path: use crate name or `crate`\n- relative path:，use self, super etc\n```rust\nfn serve_order() {}\nmod back_of_house {\n fn fix_incorrect_order() {\n cook_order();\n super::serve_order();\n }\n fn cook_order() {}\n}\n```\n```rust\nmod front_of_house {\n pub mod hosting {\n pub fn add_to_waitlist() {}\n }\n}\npub fn eat_at_restaurant() {\n // Absolute path\n crate::front_of_house::hosting::add_to_waitlist();\n // Relative path\n front_of_house::hosting::add_to_waitlist();\n}\n```\n\n### Bringing Paths into Scope with the use Keyword\n```rust\nuse std::io;\nuse std::io::Write;\nuse std::io:: { self, Write} ;\nuse std::{cmp::Ordering, io};\nuse std::fmt::Result;\nuse std::io::Result as IoResult;\nuse std::collections::*;\n\npub use crate::front_of_house::hosting; // re-exporting names with pub use\n```\n### Separating Modules into Different Files\nFilename: src/lib.rs\n```rust\nmod front_of_house;\n\npub use crate::front_of_house::hosting;\n\npub fn eat_at_restaurant() {\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n}\n```\nFilename: src/front_of_house.rs\n```rust\npub mod hosting {\n pub fn add_to_waitlist() {}\n}\n```\n\n## profile\n```\ncargo build ## dev: [unoptimized + debuginfo]\ncargo build --release ## [optimized]\n```\n- config\n```\n[profile.dev]\nopt-level = 0\n\n[profile.release]\nopt-level = 3\n\n# profile for the wasm example (from project ethers-rs)\n[profile.release.package.ethers-wasm]\nopt-level = \"s\" # Tell `rustc` to optimize for small code size.\n```\n","source":"_posts/rust/rust-08-project-management.md","raw":"---\ntitle: rust project management\ndate: 2022-11-06 15:33:55\ntags: [rust]\n---\n\n## basic concepts\n- **Packages**: A Cargo feature that lets you build, test, and share crates\n- **Crates**: A tree of modules that produces a library or executable\n- **Modules and use**: Let you control the organization, scope, and privacy of paths\n- **Paths**: A way of naming an item, such as a struct, function, or module\n\n### package\n1. one package can only has one library Crate. default is src/lib.rs\n2. one package can have multiple binary Crates (under src/bin). default src/main.rs\n3. one package has at least one crate，no matter lib or bin。\n4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)\n```\ncargo new my-project --lib ## create a library project\n```\n### crate\n- binary or library\n- The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate\n\n### module\n- keyword `mod` \n- module could be nested。\n- module includes(struct、enum、const、trait、func, etc)\n\n### path\n- absolute path: use crate name or `crate`\n- relative path:，use self, super etc\n```rust\nfn serve_order() {}\nmod back_of_house {\n fn fix_incorrect_order() {\n cook_order();\n super::serve_order();\n }\n fn cook_order() {}\n}\n```\n```rust\nmod front_of_house {\n pub mod hosting {\n pub fn add_to_waitlist() {}\n }\n}\npub fn eat_at_restaurant() {\n // Absolute path\n crate::front_of_house::hosting::add_to_waitlist();\n // Relative path\n front_of_house::hosting::add_to_waitlist();\n}\n```\n\n### Bringing Paths into Scope with the use Keyword\n```rust\nuse std::io;\nuse std::io::Write;\nuse std::io:: { self, Write} ;\nuse std::{cmp::Ordering, io};\nuse std::fmt::Result;\nuse std::io::Result as IoResult;\nuse std::collections::*;\n\npub use crate::front_of_house::hosting; // re-exporting names with pub use\n```\n### Separating Modules into Different Files\nFilename: src/lib.rs\n```rust\nmod front_of_house;\n\npub use crate::front_of_house::hosting;\n\npub fn eat_at_restaurant() {\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n}\n```\nFilename: src/front_of_house.rs\n```rust\npub mod hosting {\n pub fn add_to_waitlist() {}\n}\n```\n\n## profile\n```\ncargo build ## dev: [unoptimized + debuginfo]\ncargo build --release ## [optimized]\n```\n- config\n```\n[profile.dev]\nopt-level = 0\n\n[profile.release]\nopt-level = 3\n\n# profile for the wasm example (from project ethers-rs)\n[profile.release.package.ethers-wasm]\nopt-level = \"s\" # Tell `rustc` to optimize for small code size.\n```\n","slug":"rust/rust-08-project-management","published":1,"updated":"2023-05-03T07:41:48.892Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzq000yofsjaa7u5z4f","content":"

basic concepts

    \n
  • Packages: A Cargo feature that lets you build, test, and share crates
    • \n
  • Crates: A tree of modules that produces a library or executable
    • \n
  • Modules and use: Let you control the organization, scope, and privacy of paths
    • \n
  • Paths: A way of naming an item, such as a struct, function, or module
    • \n
\n

package

    \n
  1. one package can only has one library Crate. default is src/lib.rs
    2. \n
  2. one package can have multiple binary Crates (under src/bin). default src/main.rs
    3. \n
  3. one package has at least one crate，no matter lib or bin。
    4. \n
  4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)
    1
    cargo new my-project --lib  ## create a library project
    5. \n
\n

crate

    \n
  • binary or library
    • \n
  • The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate
    • \n
\n

module

    \n
  • keyword mod
    • \n
  • module could be nested。
    • \n
  • module includes(struct、enum、const、trait、func, etc)
    • \n
\n

path

    \n
  • absolute path: use crate name or crate
    • \n
  • relative path:，use self, super etc
    1
    2
    3
    4
    5
    6
    7
    8
    fn serve_order() {}
    mod back_of_house {
        fn fix_incorrect_order() {
            cook_order();
            super::serve_order();
        }
        fn cook_order() {}
    }
    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    mod front_of_house {
        pub mod hosting {
            pub fn add_to_waitlist() {}
        }
    }
    pub fn eat_at_restaurant() {
        // Absolute path
        crate::front_of_house::hosting::add_to_waitlist();
        // Relative path
        front_of_house::hosting::add_to_waitlist();
    }
    • \n
\n

Bringing Paths into Scope with the use Keyword

1
2
3
4
5
6
7
8
9
use std::io;
use std::io::Write;
use std::io:: { self, Write} ;
use std::{cmp::Ordering, io};
use std::fmt::Result;
use std::io::Result as IoResult;
use std::collections::*;

pub use crate::front_of_house::hosting; // re-exporting names with pub use
\n

Separating Modules into Different Files

Filename: src/lib.rs

\n
1
2
3
4
5
6
7
8
9
mod front_of_house;

pub use crate::front_of_house::hosting;

pub fn eat_at_restaurant() {
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
}
\n

Filename: src/front_of_house.rs

\n
1
2
3
pub mod hosting {
    pub fn add_to_waitlist() {}
}
\n\n

profile

1
2
cargo build ## dev: [unoptimized + debuginfo]
cargo build --release ## [optimized]
\n
    \n
  • config
    1
    2
    3
    4
    5
    6
    7
    8
    9
    [profile.dev]
    opt-level = 0

    [profile.release]
    opt-level = 3

    # profile for the wasm example (from project ethers-rs)
    [profile.release.package.ethers-wasm]
    opt-level = "s"  # Tell `rustc` to optimize for small code size.
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

basic concepts

    \n
  • Packages: A Cargo feature that lets you build, test, and share crates
    • \n
  • Crates: A tree of modules that produces a library or executable
    • \n
  • Modules and use: Let you control the organization, scope, and privacy of paths
    • \n
  • Paths: A way of naming an item, such as a struct, function, or module
    • \n
\n

package

    \n
  1. one package can only has one library Crate. default is src/lib.rs
    2. \n
  2. one package can have multiple binary Crates (under src/bin). default src/main.rs
    3. \n
  3. one package has at least one crate，no matter lib or bin。
    4. \n
  4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)
    1
    cargo new my-project --lib  ## create a library project
    5. \n
\n

crate

    \n
  • binary or library
    • \n
  • The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate
    • \n
\n

module

    \n
  • keyword mod
    • \n
  • module could be nested。
    • \n
  • module includes(struct、enum、const、trait、func, etc)
    • \n
\n

path

    \n
  • absolute path: use crate name or crate
    • \n
  • relative path:，use self, super etc
    1
    2
    3
    4
    5
    6
    7
    8
    fn serve_order() {}
    mod back_of_house {
        fn fix_incorrect_order() {
            cook_order();
            super::serve_order();
        }
        fn cook_order() {}
    }
    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    mod front_of_house {
        pub mod hosting {
            pub fn add_to_waitlist() {}
        }
    }
    pub fn eat_at_restaurant() {
        // Absolute path
        crate::front_of_house::hosting::add_to_waitlist();
        // Relative path
        front_of_house::hosting::add_to_waitlist();
    }
    • \n
\n

Bringing Paths into Scope with the use Keyword

1
2
3
4
5
6
7
8
9
use std::io;
use std::io::Write;
use std::io:: { self, Write} ;
use std::{cmp::Ordering, io};
use std::fmt::Result;
use std::io::Result as IoResult;
use std::collections::*;

pub use crate::front_of_house::hosting; // re-exporting names with pub use
\n

Separating Modules into Different Files

Filename: src/lib.rs

\n
1
2
3
4
5
6
7
8
9
mod front_of_house;

pub use crate::front_of_house::hosting;

pub fn eat_at_restaurant() {
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
}
\n

Filename: src/front_of_house.rs

\n
1
2
3
pub mod hosting {
    pub fn add_to_waitlist() {}
}
\n\n

profile

1
2
cargo build ## dev: [unoptimized + debuginfo]
cargo build --release ## [optimized]
\n
    \n
  • config
    1
    2
    3
    4
    5
    6
    7
    8
    9
    [profile.dev]
    opt-level = 0

    [profile.release]
    opt-level = 3

    # profile for the wasm example (from project ethers-rs)
    [profile.release.package.ethers-wasm]
    opt-level = "s"  # Tell `rustc` to optimize for small code size.
    • \n
\n"},{"title":"rust smart pointer","date":"2022-10-30T03:00:38.000Z","_content":"\n## Overview\nThe most common kind of pointer in Rust is a reference (borrow but not own)\nSmart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.\nreferences are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.\n\n### smart pointers example\n- String\n- Vec\nBoth these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).\n\n### Deref & Drop\nSmart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the `Deref` and `Drop` traits.\n- The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers. \n- The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.\n\n### the most common smart pointers in the standard library:\n- `Box` for allocating values on the heap\n- `Rc`, a reference counting type that enables multiple ownership\n- `Ref` and `RefMut`, accessed through `RefCell`, a type that enforces the borrowing rules at runtime instead of compile time\n\n## Box\n### when to use\n- When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)\n- When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so\n- When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type\n\n\n### enabling recursive types with Box\nAt compile time, Rust needs to know how much space a type takes up\nOne type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address\n\n```rust\nenum List {\n Cons(i32, List),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nfn main() {\n let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size\n}\n```\n\nuse Box\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));\n}\n```\n\n## Treating Smart Pointers Like Regular References with the Deref Trait\nImplementing the Deref trait allows you to customize the behavior of the dereference operator, `*`\nBy implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.\n\n### Defining Our Own Smart Pointer\n```rust\nuse std::ops::Deref;\n\nstruct MyBox(T); // The MyBox type is a tuple struct with one element of type T\n\nimpl MyBox {\n fn new(x: T) -> MyBox {\n MyBox(x)\n }\n}\n\nimpl Deref for MyBox {\n type Target = T;\n\n fn deref(&self) -> &Self::Target {\n &self.0\n }\n}\n\nfn main() {\n let x = 5;\n let y = MyBox::new(x);\n\n assert_eq!(5, x);\n assert_eq!(5, *y);\n}\n```\n- We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator. \n- behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method\n- The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.\n\n### Implicit Deref Coercions with Functions and Methods\nDeref coercion is a convenience that Rust performs on arguments to functions and methods. \nDeref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str\nA sequence of calls to the deref method converts the type we provided into the type the parameter needs.\n\n```rust\nfn hello(name: &str) {\n println!(\"Hello, {}!\", name);\n}\n\nfn main() {\n let m = MyBox::new(String::from(\"Rust\"));\n hello(\"rust\"); // ok\n hello(&m); // also ok\n hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello\n}\n```\nHere we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.\n\n### How Deref Coercion Interacts with Mutability\nSimilar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.\n\nRust does deref coercion when it finds types and trait implementations in three cases:\n- From &T to &U when T: Deref\n- From &mut T to &mut U when T: DerefMut\n- From &mut T to &U when T: Deref\n\n## Running Code on Cleanup with the Drop Trait\nDrop, which lets you customize what happens when a value is about to go out of scope. \n\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"my stuff\"),\n };\n c.drop(); // not allowed\n let d = CustomSmartPointer {\n data: String::from(\"other stuff\"),\n };\n println!(\"CustomSmartPointers created.\");\n}\n\n```\n### Dropping a Value Early with std::mem::drop\nstd::mem::drop is in prelude, can use directly\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"some data\"),\n };\n println!(\"CustomSmartPointer created.\");\n drop(c); // ok\n println!(\"CustomSmartPointer dropped before the end of main.\");\n} // c goes out of scope, will occur double drop\n\n```\n\n\n## Rc: the Reference Counted Smart Pointer\nIn the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners. \ntype keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.\n\nWe use the `Rc` type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.\n\nRc is only for use in single-threaded scenarios\n\n### using Rc to share data\n\n![rc](/images/rust/pointers/rc.png)\nimplement use box will not work, as below\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));\n let b = Cons(3, Box::new(a)); // value moved here\n let c = Cons(4, Box::new(a)); // value used here after move\n}\n```\nuse Rc\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data\n let c = Cons(4, Rc::clone(&a));\n}\n```\n\n\nWe could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.\n\n### Cloning an Rc Increases the Reference Count\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n println!(\"count after creating a = {}\", Rc::strong_count(&a)); // 1\n let b = Cons(3, Rc::clone(&a));\n println!(\"count after creating b = {}\", Rc::strong_count(&a)); // 2\n {\n let c = Cons(4, Rc::clone(&a));\n println!(\"count after creating c = {}\", Rc::strong_count(&a)); // 3\n }\n println!(\"count after c goes out of scope = {}\", Rc::strong_count(&a)); // 2\n}\n```\nWhat we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.\n\n## RefCell and interior mutability pattern\nInterior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.\nTo mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.\nWe can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that. \nThe unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.\n\n### Enforcing Borrowing Rules at Runtime with RefCell\nUnlike Rc, the RefCell type represents single ownership over the data it holds.\n\nrecall borrowing rules\n- At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.\n- References must always be valid.\nWith references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime. \nBecause RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.\n\n### Interior Mutability: A Mutable Borrow to an Immutable Value\n```rust\nfn main() {\n let x = 5;\n let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable\n}\n```\n\n### A Use Case for Interior Mutability: Mock Objects\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n```\na problematic usage\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n struct MockMessenger {\n sent_messages: Vec,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: vec![],\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.len(), 1);\n }\n}\n\n```\nWe can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition\n\nThis is a situation in which interior mutability can help! \n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::cell::RefCell;\n\n struct MockMessenger {\n sent_messages: RefCell>,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: RefCell::new(vec![]),\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell> in self.sent_messages to get a mutable reference to the value inside the RefCell>\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n // --snip--\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); // call borrow on the RefCell> to get an immutable reference to the vector.\n }\n}\n```\n\n### Keeping Track of Borrows at Runtime with RefCell\nWhen creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. **The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut**. Both types implement Deref, so we can treat them like regular references.\n\nThe RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime. \n```rust\nimpl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n let mut one_borrow = self.sent_messages.borrow_mut();\n let mut two_borrow = self.sent_messages.borrow_mut();\n\n one_borrow.push(String::from(message));\n two_borrow.push(String::from(message));\n }\n }\n```\nWhen we run the tests for our library, the code in will compile without any errors, but the test will fail\nthread 'main' panicked at 'already borrowed\n\n### Having Multiple Owners of Mutable Data by Combining Rc and RefCell\nA common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!\n```rust\n#[derive(Debug)]\nenum List {\n Cons(Rc>, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\nfn main() {\n let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc> and store it in a variable named value so we can access it directly later.\n\n let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); // Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc so when we create lists b and c, they can both refer to a\n\n let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));\n let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));\n\n *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc to the inner RefCell value. The borrow_mut method returns a RefMut smart pointer, and we use the dereference operator on it and change the inner value.\n\n println!(\"a after = {:?}\", a);\n println!(\"b after = {:?}\", b);\n println!(\"c after = {:?}\", c);\n}\n```\n\n**The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;** \n\n## Reference Cycles Can Leak Memory\nRust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).\n\n### Creating a Reference Cycle\n```rust\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>), // The second element in the Cons variant is now RefCell>, meaning that we want to modify which List value a Cons variant is pointing to. \n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n use crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>),\n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> {\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));\n\n println!(\"a initial rc count = {}\", Rc::strong_count(&a));\n println!(\"a next item = {:?}\", a.tail());\n\n let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a\n\n println!(\"a rc count after b creation = {}\", Rc::strong_count(&a));\n println!(\"b initial rc count = {}\", Rc::strong_count(&b));\n println!(\"b next item = {:?}\", b.tail());\n\n if let Some(link) = a.tail() {\n *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle\n }\n\n println!(\"b rc count after changing a = {}\", Rc::strong_count(&b));\n println!(\"a rc count after changing a = {}\", Rc::strong_count(&a));\n\n // Uncomment the next line to see that we have a cycle;\n // it will overflow the stack\n // println!(\"a next item = {:?}\", a.tail());\n} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc instance from 2 to 1. The memory that Rc has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc instance still refers to it. The memory allocated to the list will remain uncollected forever.\n```\n![cycle-ref](/images/rust/pointers/cycle.ref.png)\n\n### Preventing Reference Cycles: Turning an Rc into a Weak\nSo far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.\n\nStrong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.\n\nBecause the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped. \n\n\n### Creating a Tree Data Structure: a Node with Child Nodes\n```rust\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.\n children: RefCell>>,\n}\n\nfn main() {\n\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. \n\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. \n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak reference to branch from the Rc in branch.\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n\n \n}\n```\n\n### Visualizing Changes to strong_count and weak_count\n```rust\nuse std::cell::RefCell;\nuse std::rc::{Rc, Weak};\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>,\n children: RefCell>>,\n}\n\nfn main() {\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n\n {\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]),\n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch);\n\n println!(\n \"branch strong = {}, weak = {}\",\n Rc::strong_count(&branch),\n Rc::weak_count(&branch),\n );\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n }\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n}\n```","source":"_posts/rust/rust-06-smart-pointer.md","raw":"---\ntitle: rust smart pointer\ndate: 2022-10-30 11:00:38\ntags: [rust]\n---\n\n## Overview\nThe most common kind of pointer in Rust is a reference (borrow but not own)\nSmart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.\nreferences are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.\n\n### smart pointers example\n- String\n- Vec\nBoth these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).\n\n### Deref & Drop\nSmart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the `Deref` and `Drop` traits.\n- The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers. \n- The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.\n\n### the most common smart pointers in the standard library:\n- `Box` for allocating values on the heap\n- `Rc`, a reference counting type that enables multiple ownership\n- `Ref` and `RefMut`, accessed through `RefCell`, a type that enforces the borrowing rules at runtime instead of compile time\n\n## Box\n### when to use\n- When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)\n- When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so\n- When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type\n\n\n### enabling recursive types with Box\nAt compile time, Rust needs to know how much space a type takes up\nOne type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address\n\n```rust\nenum List {\n Cons(i32, List),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nfn main() {\n let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size\n}\n```\n\nuse Box\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));\n}\n```\n\n## Treating Smart Pointers Like Regular References with the Deref Trait\nImplementing the Deref trait allows you to customize the behavior of the dereference operator, `*`\nBy implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.\n\n### Defining Our Own Smart Pointer\n```rust\nuse std::ops::Deref;\n\nstruct MyBox(T); // The MyBox type is a tuple struct with one element of type T\n\nimpl MyBox {\n fn new(x: T) -> MyBox {\n MyBox(x)\n }\n}\n\nimpl Deref for MyBox {\n type Target = T;\n\n fn deref(&self) -> &Self::Target {\n &self.0\n }\n}\n\nfn main() {\n let x = 5;\n let y = MyBox::new(x);\n\n assert_eq!(5, x);\n assert_eq!(5, *y);\n}\n```\n- We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator. \n- behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method\n- The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.\n\n### Implicit Deref Coercions with Functions and Methods\nDeref coercion is a convenience that Rust performs on arguments to functions and methods. \nDeref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str\nA sequence of calls to the deref method converts the type we provided into the type the parameter needs.\n\n```rust\nfn hello(name: &str) {\n println!(\"Hello, {}!\", name);\n}\n\nfn main() {\n let m = MyBox::new(String::from(\"Rust\"));\n hello(\"rust\"); // ok\n hello(&m); // also ok\n hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello\n}\n```\nHere we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.\n\n### How Deref Coercion Interacts with Mutability\nSimilar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.\n\nRust does deref coercion when it finds types and trait implementations in three cases:\n- From &T to &U when T: Deref\n- From &mut T to &mut U when T: DerefMut\n- From &mut T to &U when T: Deref\n\n## Running Code on Cleanup with the Drop Trait\nDrop, which lets you customize what happens when a value is about to go out of scope. \n\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"my stuff\"),\n };\n c.drop(); // not allowed\n let d = CustomSmartPointer {\n data: String::from(\"other stuff\"),\n };\n println!(\"CustomSmartPointers created.\");\n}\n\n```\n### Dropping a Value Early with std::mem::drop\nstd::mem::drop is in prelude, can use directly\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"some data\"),\n };\n println!(\"CustomSmartPointer created.\");\n drop(c); // ok\n println!(\"CustomSmartPointer dropped before the end of main.\");\n} // c goes out of scope, will occur double drop\n\n```\n\n\n## Rc: the Reference Counted Smart Pointer\nIn the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners. \ntype keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.\n\nWe use the `Rc` type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.\n\nRc is only for use in single-threaded scenarios\n\n### using Rc to share data\n\n![rc](/images/rust/pointers/rc.png)\nimplement use box will not work, as below\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));\n let b = Cons(3, Box::new(a)); // value moved here\n let c = Cons(4, Box::new(a)); // value used here after move\n}\n```\nuse Rc\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data\n let c = Cons(4, Rc::clone(&a));\n}\n```\n\n\nWe could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.\n\n### Cloning an Rc Increases the Reference Count\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n println!(\"count after creating a = {}\", Rc::strong_count(&a)); // 1\n let b = Cons(3, Rc::clone(&a));\n println!(\"count after creating b = {}\", Rc::strong_count(&a)); // 2\n {\n let c = Cons(4, Rc::clone(&a));\n println!(\"count after creating c = {}\", Rc::strong_count(&a)); // 3\n }\n println!(\"count after c goes out of scope = {}\", Rc::strong_count(&a)); // 2\n}\n```\nWhat we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.\n\n## RefCell and interior mutability pattern\nInterior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.\nTo mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.\nWe can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that. \nThe unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.\n\n### Enforcing Borrowing Rules at Runtime with RefCell\nUnlike Rc, the RefCell type represents single ownership over the data it holds.\n\nrecall borrowing rules\n- At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.\n- References must always be valid.\nWith references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime. \nBecause RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.\n\n### Interior Mutability: A Mutable Borrow to an Immutable Value\n```rust\nfn main() {\n let x = 5;\n let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable\n}\n```\n\n### A Use Case for Interior Mutability: Mock Objects\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n```\na problematic usage\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n struct MockMessenger {\n sent_messages: Vec,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: vec![],\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.len(), 1);\n }\n}\n\n```\nWe can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition\n\nThis is a situation in which interior mutability can help! \n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::cell::RefCell;\n\n struct MockMessenger {\n sent_messages: RefCell>,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: RefCell::new(vec![]),\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell> in self.sent_messages to get a mutable reference to the value inside the RefCell>\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n // --snip--\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); // call borrow on the RefCell> to get an immutable reference to the vector.\n }\n}\n```\n\n### Keeping Track of Borrows at Runtime with RefCell\nWhen creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. **The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut**. Both types implement Deref, so we can treat them like regular references.\n\nThe RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime. \n```rust\nimpl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n let mut one_borrow = self.sent_messages.borrow_mut();\n let mut two_borrow = self.sent_messages.borrow_mut();\n\n one_borrow.push(String::from(message));\n two_borrow.push(String::from(message));\n }\n }\n```\nWhen we run the tests for our library, the code in will compile without any errors, but the test will fail\nthread 'main' panicked at 'already borrowed\n\n### Having Multiple Owners of Mutable Data by Combining Rc and RefCell\nA common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!\n```rust\n#[derive(Debug)]\nenum List {\n Cons(Rc>, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\nfn main() {\n let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc> and store it in a variable named value so we can access it directly later.\n\n let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); // Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc so when we create lists b and c, they can both refer to a\n\n let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));\n let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));\n\n *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc to the inner RefCell value. The borrow_mut method returns a RefMut smart pointer, and we use the dereference operator on it and change the inner value.\n\n println!(\"a after = {:?}\", a);\n println!(\"b after = {:?}\", b);\n println!(\"c after = {:?}\", c);\n}\n```\n\n**The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;** \n\n## Reference Cycles Can Leak Memory\nRust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).\n\n### Creating a Reference Cycle\n```rust\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>), // The second element in the Cons variant is now RefCell>, meaning that we want to modify which List value a Cons variant is pointing to. \n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n use crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>),\n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> {\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));\n\n println!(\"a initial rc count = {}\", Rc::strong_count(&a));\n println!(\"a next item = {:?}\", a.tail());\n\n let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a\n\n println!(\"a rc count after b creation = {}\", Rc::strong_count(&a));\n println!(\"b initial rc count = {}\", Rc::strong_count(&b));\n println!(\"b next item = {:?}\", b.tail());\n\n if let Some(link) = a.tail() {\n *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle\n }\n\n println!(\"b rc count after changing a = {}\", Rc::strong_count(&b));\n println!(\"a rc count after changing a = {}\", Rc::strong_count(&a));\n\n // Uncomment the next line to see that we have a cycle;\n // it will overflow the stack\n // println!(\"a next item = {:?}\", a.tail());\n} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc instance from 2 to 1. The memory that Rc has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc instance still refers to it. The memory allocated to the list will remain uncollected forever.\n```\n![cycle-ref](/images/rust/pointers/cycle.ref.png)\n\n### Preventing Reference Cycles: Turning an Rc into a Weak\nSo far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.\n\nStrong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.\n\nBecause the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped. \n\n\n### Creating a Tree Data Structure: a Node with Child Nodes\n```rust\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.\n children: RefCell>>,\n}\n\nfn main() {\n\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. \n\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. \n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak reference to branch from the Rc in branch.\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n\n \n}\n```\n\n### Visualizing Changes to strong_count and weak_count\n```rust\nuse std::cell::RefCell;\nuse std::rc::{Rc, Weak};\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>,\n children: RefCell>>,\n}\n\nfn main() {\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n\n {\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]),\n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch);\n\n println!(\n \"branch strong = {}, weak = {}\",\n Rc::strong_count(&branch),\n Rc::weak_count(&branch),\n );\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n }\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n}\n```","slug":"rust/rust-06-smart-pointer","published":1,"updated":"2023-05-03T07:31:43.613Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzq0010ofsjfj0gc4g6","content":"

Overview

The most common kind of pointer in Rust is a reference (borrow but not own)
Smart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.
references are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.

\n

smart pointers example

    \n
  • String
    • \n
  • Vec
    Both these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).
    • \n
\n

Deref & Drop

Smart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the Deref and Drop traits.

\n
    \n
  • The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers.
    • \n
  • The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.
    • \n
\n

the most common smart pointers in the standard library:

    \n
  • Box<T> for allocating values on the heap
    • \n
  • Rc<T>, a reference counting type that enables multiple ownership
    • \n
  • Ref<T> and RefMut<T>, accessed through RefCell<T>, a type that enforces the borrowing rules at runtime instead of compile time
    • \n
\n

Box

when to use

    \n
  • When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)
    • \n
  • When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so
    • \n
  • When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type
    • \n
\n

enabling recursive types with Box

At compile time, Rust needs to know how much space a type takes up
One type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address

\n
1
2
3
4
5
6
7
8
9
enum List {
    Cons(i32, List),
    Nil,
}

use crate::List::{Cons, Nil};
fn main() {
    let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size
}
\n\n

use Box

\n
1
2
3
4
5
6
7
8
9
10
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));
}
\n\n

Treating Smart Pointers Like Regular References with the Deref Trait

Implementing the Deref trait allows you to customize the behavior of the dereference operator, *
By implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.

\n

Defining Our Own Smart Pointer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
use std::ops::Deref;

struct MyBox<T>(T);  // The MyBox type is a tuple struct with one element of type T

impl<T> MyBox<T> {
    fn new(x: T) -> MyBox<T> {
        MyBox(x)
    }
}

impl<T> Deref for MyBox<T> {
    type Target = T;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

fn main() {
    let x = 5;
    let y = MyBox::new(x);

    assert_eq!(5, x);
    assert_eq!(5, *y);
}
\n
    \n
  • We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator.
    • \n
  • behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method
    • \n
  • The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.
    • \n
\n

Implicit Deref Coercions with Functions and Methods

Deref coercion is a convenience that Rust performs on arguments to functions and methods.
Deref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str
A sequence of calls to the deref method converts the type we provided into the type the parameter needs.

\n
1
2
3
4
5
6
7
8
9
10
fn hello(name: &str) {
    println!("Hello, {}!", name);
}

fn main() {
    let m = MyBox::new(String::from("Rust"));
    hello("rust");  // ok
    hello(&m);      // also ok
    hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox<String> into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello
}
\n

Here we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.

\n

How Deref Coercion Interacts with Mutability

Similar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.

\n

Rust does deref coercion when it finds types and trait implementations in three cases:

\n
    \n
  • From &T to &U when T: Deref<Target=U>
    • \n
  • From &mut T to &mut U when T: DerefMut<Target=U>
    • \n
  • From &mut T to &U when T: Deref<Target=U>
    • \n
\n

Running Code on Cleanup with the Drop Trait

Drop, which lets you customize what happens when a value is about to go out of scope.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("my stuff"),
    };
    c.drop(); // not allowed
    let d = CustomSmartPointer {
        data: String::from("other stuff"),
    };
    println!("CustomSmartPointers created.");
}

\n

Dropping a Value Early with std::mem::drop

std::mem::drop is in prelude, can use directly

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("some data"),
    };
    println!("CustomSmartPointer created.");
    drop(c); // ok
    println!("CustomSmartPointer dropped before the end of main.");
} // c goes out of scope, will occur double drop

\n\n\n

Rc: the Reference Counted Smart Pointer

In the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners.
type keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.

\n

We use the Rc<T> type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.

\n

Rc is only for use in single-threaded scenarios

\n

using Rc to share data

\"rc\"
implement use box will not work, as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));
    let b = Cons(3, Box::new(a)); // value moved here
    let c = Cons(4, Box::new(a)); // value used here after move
}
\n

use Rc

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data
    let c = Cons(4, Rc::clone(&a));
}
\n\n\n

We could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.

\n

Cloning an Rc Increases the Reference Count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    println!("count after creating a = {}", Rc::strong_count(&a));  // 1
    let b = Cons(3, Rc::clone(&a));
    println!("count after creating b = {}", Rc::strong_count(&a));  // 2
    {
        let c = Cons(4, Rc::clone(&a));
        println!("count after creating c = {}", Rc::strong_count(&a)); // 3
    }
    println!("count after c goes out of scope = {}", Rc::strong_count(&a));  // 2
}
\n

What we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.

\n

RefCell and interior mutability pattern

Interior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.
To mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.
We can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that.
The unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.

\n

Enforcing Borrowing Rules at Runtime with RefCell

Unlike Rc, the RefCell type represents single ownership over the data it holds.

\n

recall borrowing rules

\n
    \n
  • At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.
    • \n
  • References must always be valid.
    With references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime.
    Because RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.
    • \n
\n

Interior Mutability: A Mutable Borrow to an Immutable Value

1
2
3
4
fn main() {
    let x = 5;
    let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable
}
\n\n

A Use Case for Interior Mutability: Mock Objects

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

\n

a problematic usage

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    struct MockMessenger {
        sent_messages: Vec<String>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: vec![],
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.len(), 1);
    }
}

\n

We can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition

\n

This is a situation in which interior mutability can help!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::RefCell;

    struct MockMessenger {
        sent_messages: RefCell<Vec<String>>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: RefCell::new(vec![]),
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell<Vec<String>> in self.sent_messages to get a mutable reference to the value inside the RefCell<Vec<String>>
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        // --snip--
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); //  call borrow on the RefCell<Vec<String>> to get an immutable reference to the vector.
    }
}
\n\n

Keeping Track of Borrows at Runtime with RefCell

When creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut. Both types implement Deref, so we can treat them like regular references.

\n

The RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime.

\n
1
2
3
4
5
6
7
8
9
impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            let mut one_borrow = self.sent_messages.borrow_mut();
            let mut two_borrow = self.sent_messages.borrow_mut();

            one_borrow.push(String::from(message));
            two_borrow.push(String::from(message));
        }
    }
\n

When we run the tests for our library, the code in will compile without any errors, but the test will fail
thread ‘main’ panicked at ‘already borrowed

\n

Having Multiple Owners of Mutable Data by Combining Rc and RefCell

A common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#[derive(Debug)]
enum List {
    Cons(Rc<RefCell<i32>>, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

fn main() {
    let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc<RefCell<i32>> and store it in a variable named value so we can access it directly later.

    let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); //  Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc<T> so when we create lists b and c, they can both refer to a

    let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));
    let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));

    *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc<T> to the inner RefCell<T> value. The borrow_mut method returns a RefMut<T> smart pointer, and we use the dereference operator on it and change the inner value.

    println!("a after = {:?}", a);
    println!("b after = {:?}", b);
    println!("c after = {:?}", c);
}
\n\n

The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;

\n

Reference Cycles Can Leak Memory

Rust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).

\n

Creating a Reference Cycle

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>), // The second element in the Cons variant is now RefCell<Rc<List>>, meaning that we want to modify which List value a Cons variant is pointing to. 
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>),
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> {
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));

    println!("a initial rc count = {}", Rc::strong_count(&a));
    println!("a next item = {:?}", a.tail());

    let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a

    println!("a rc count after b creation = {}", Rc::strong_count(&a));
    println!("b initial rc count = {}", Rc::strong_count(&b));
    println!("b next item = {:?}", b.tail());

    if let Some(link) = a.tail() {
        *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle
    }

    println!("b rc count after changing a = {}", Rc::strong_count(&b));
    println!("a rc count after changing a = {}", Rc::strong_count(&a));

    // Uncomment the next line to see that we have a cycle;
    // it will overflow the stack
    // println!("a next item = {:?}", a.tail());
} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc<List> instance from 2 to 1. The memory that Rc<List> has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc<List> instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc<List> instance still refers to it. The memory allocated to the list will remain uncollected forever.
\n

\"cycle-ref\"

\n

Preventing Reference Cycles: Turning an Rc into a Weak

So far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.

\n

Strong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.

\n

Because the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option<Rc>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped.

\n

Creating a Tree Data Structure: a Node with Child Nodes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc<T>, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {

    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. 

    let branch = Rc::new(Node {
        value: 5,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc<Node> in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. 
    });

    *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak<Node> reference to branch from the Rc<Node> in branch.

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());

   
}
\n\n

Visualizing Changes to strong_count and weak_count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
use std::cell::RefCell;
use std::rc::{Rc, Weak};

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>,
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {
    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );

    {
        let branch = Rc::new(Node {
            value: 5,
            parent: RefCell::new(Weak::new()),
            children: RefCell::new(vec![Rc::clone(&leaf)]),
        });

        *leaf.parent.borrow_mut() = Rc::downgrade(&branch);

        println!(
            "branch strong = {}, weak = {}",
            Rc::strong_count(&branch),
            Rc::weak_count(&branch),
        );

        println!(
            "leaf strong = {}, weak = {}",
            Rc::strong_count(&leaf),
            Rc::weak_count(&leaf),
        );
    }

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());
    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );
}
","site":{"data":{}},"excerpt":"","more":"

Overview

The most common kind of pointer in Rust is a reference (borrow but not own)
Smart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.
references are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.

\n

smart pointers example

    \n
  • String
    • \n
  • Vec
    Both these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).
    • \n
\n

Deref & Drop

Smart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the Deref and Drop traits.

\n
    \n
  • The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers.
    • \n
  • The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.
    • \n
\n

the most common smart pointers in the standard library:

    \n
  • Box<T> for allocating values on the heap
    • \n
  • Rc<T>, a reference counting type that enables multiple ownership
    • \n
  • Ref<T> and RefMut<T>, accessed through RefCell<T>, a type that enforces the borrowing rules at runtime instead of compile time
    • \n
\n

Box

when to use

    \n
  • When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)
    • \n
  • When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so
    • \n
  • When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type
    • \n
\n

enabling recursive types with Box

At compile time, Rust needs to know how much space a type takes up
One type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address

\n
1
2
3
4
5
6
7
8
9
enum List {
    Cons(i32, List),
    Nil,
}

use crate::List::{Cons, Nil};
fn main() {
    let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size
}
\n\n

use Box

\n
1
2
3
4
5
6
7
8
9
10
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));
}
\n\n

Treating Smart Pointers Like Regular References with the Deref Trait

Implementing the Deref trait allows you to customize the behavior of the dereference operator, *
By implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.

\n

Defining Our Own Smart Pointer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
use std::ops::Deref;

struct MyBox<T>(T);  // The MyBox type is a tuple struct with one element of type T

impl<T> MyBox<T> {
    fn new(x: T) -> MyBox<T> {
        MyBox(x)
    }
}

impl<T> Deref for MyBox<T> {
    type Target = T;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

fn main() {
    let x = 5;
    let y = MyBox::new(x);

    assert_eq!(5, x);
    assert_eq!(5, *y);
}
\n
    \n
  • We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator.
    • \n
  • behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method
    • \n
  • The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.
    • \n
\n

Implicit Deref Coercions with Functions and Methods

Deref coercion is a convenience that Rust performs on arguments to functions and methods.
Deref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str
A sequence of calls to the deref method converts the type we provided into the type the parameter needs.

\n
1
2
3
4
5
6
7
8
9
10
fn hello(name: &str) {
    println!("Hello, {}!", name);
}

fn main() {
    let m = MyBox::new(String::from("Rust"));
    hello("rust");  // ok
    hello(&m);      // also ok
    hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox<String> into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello
}
\n

Here we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.

\n

How Deref Coercion Interacts with Mutability

Similar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.

\n

Rust does deref coercion when it finds types and trait implementations in three cases:

\n
    \n
  • From &T to &U when T: Deref<Target=U>
    • \n
  • From &mut T to &mut U when T: DerefMut<Target=U>
    • \n
  • From &mut T to &U when T: Deref<Target=U>
    • \n
\n

Running Code on Cleanup with the Drop Trait

Drop, which lets you customize what happens when a value is about to go out of scope.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("my stuff"),
    };
    c.drop(); // not allowed
    let d = CustomSmartPointer {
        data: String::from("other stuff"),
    };
    println!("CustomSmartPointers created.");
}

\n

Dropping a Value Early with std::mem::drop

std::mem::drop is in prelude, can use directly

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("some data"),
    };
    println!("CustomSmartPointer created.");
    drop(c); // ok
    println!("CustomSmartPointer dropped before the end of main.");
} // c goes out of scope, will occur double drop

\n\n\n

Rc: the Reference Counted Smart Pointer

In the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners.
type keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.

\n

We use the Rc<T> type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.

\n

Rc is only for use in single-threaded scenarios

\n

using Rc to share data

\"rc\"
implement use box will not work, as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));
    let b = Cons(3, Box::new(a)); // value moved here
    let c = Cons(4, Box::new(a)); // value used here after move
}
\n

use Rc

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data
    let c = Cons(4, Rc::clone(&a));
}
\n\n\n

We could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.

\n

Cloning an Rc Increases the Reference Count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    println!("count after creating a = {}", Rc::strong_count(&a));  // 1
    let b = Cons(3, Rc::clone(&a));
    println!("count after creating b = {}", Rc::strong_count(&a));  // 2
    {
        let c = Cons(4, Rc::clone(&a));
        println!("count after creating c = {}", Rc::strong_count(&a)); // 3
    }
    println!("count after c goes out of scope = {}", Rc::strong_count(&a));  // 2
}
\n

What we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.

\n

RefCell and interior mutability pattern

Interior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.
To mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.
We can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that.
The unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.

\n

Enforcing Borrowing Rules at Runtime with RefCell

Unlike Rc, the RefCell type represents single ownership over the data it holds.

\n

recall borrowing rules

\n
    \n
  • At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.
    • \n
  • References must always be valid.
    With references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime.
    Because RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.
    • \n
\n

Interior Mutability: A Mutable Borrow to an Immutable Value

1
2
3
4
fn main() {
    let x = 5;
    let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable
}
\n\n

A Use Case for Interior Mutability: Mock Objects

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

\n

a problematic usage

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    struct MockMessenger {
        sent_messages: Vec<String>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: vec![],
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.len(), 1);
    }
}

\n

We can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition

\n

This is a situation in which interior mutability can help!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::RefCell;

    struct MockMessenger {
        sent_messages: RefCell<Vec<String>>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: RefCell::new(vec![]),
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell<Vec<String>> in self.sent_messages to get a mutable reference to the value inside the RefCell<Vec<String>>
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        // --snip--
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); //  call borrow on the RefCell<Vec<String>> to get an immutable reference to the vector.
    }
}
\n\n

Keeping Track of Borrows at Runtime with RefCell

When creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut. Both types implement Deref, so we can treat them like regular references.

\n

The RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime.

\n
1
2
3
4
5
6
7
8
9
impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            let mut one_borrow = self.sent_messages.borrow_mut();
            let mut two_borrow = self.sent_messages.borrow_mut();

            one_borrow.push(String::from(message));
            two_borrow.push(String::from(message));
        }
    }
\n

When we run the tests for our library, the code in will compile without any errors, but the test will fail
thread ‘main’ panicked at ‘already borrowed

\n

Having Multiple Owners of Mutable Data by Combining Rc and RefCell

A common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#[derive(Debug)]
enum List {
    Cons(Rc<RefCell<i32>>, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

fn main() {
    let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc<RefCell<i32>> and store it in a variable named value so we can access it directly later.

    let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); //  Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc<T> so when we create lists b and c, they can both refer to a

    let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));
    let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));

    *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc<T> to the inner RefCell<T> value. The borrow_mut method returns a RefMut<T> smart pointer, and we use the dereference operator on it and change the inner value.

    println!("a after = {:?}", a);
    println!("b after = {:?}", b);
    println!("c after = {:?}", c);
}
\n\n

The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;

\n

Reference Cycles Can Leak Memory

Rust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).

\n

Creating a Reference Cycle

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>), // The second element in the Cons variant is now RefCell<Rc<List>>, meaning that we want to modify which List value a Cons variant is pointing to. 
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>),
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> {
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));

    println!("a initial rc count = {}", Rc::strong_count(&a));
    println!("a next item = {:?}", a.tail());

    let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a

    println!("a rc count after b creation = {}", Rc::strong_count(&a));
    println!("b initial rc count = {}", Rc::strong_count(&b));
    println!("b next item = {:?}", b.tail());

    if let Some(link) = a.tail() {
        *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle
    }

    println!("b rc count after changing a = {}", Rc::strong_count(&b));
    println!("a rc count after changing a = {}", Rc::strong_count(&a));

    // Uncomment the next line to see that we have a cycle;
    // it will overflow the stack
    // println!("a next item = {:?}", a.tail());
} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc<List> instance from 2 to 1. The memory that Rc<List> has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc<List> instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc<List> instance still refers to it. The memory allocated to the list will remain uncollected forever.
\n

\"cycle-ref\"

\n

Preventing Reference Cycles: Turning an Rc into a Weak

So far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.

\n

Strong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.

\n

Because the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option<Rc>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped.

\n

Creating a Tree Data Structure: a Node with Child Nodes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc<T>, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {

    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. 

    let branch = Rc::new(Node {
        value: 5,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc<Node> in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. 
    });

    *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak<Node> reference to branch from the Rc<Node> in branch.

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());

   
}
\n\n

Visualizing Changes to strong_count and weak_count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
use std::cell::RefCell;
use std::rc::{Rc, Weak};

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>,
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {
    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );

    {
        let branch = Rc::new(Node {
            value: 5,
            parent: RefCell::new(Weak::new()),
            children: RefCell::new(vec![Rc::clone(&leaf)]),
        });

        *leaf.parent.borrow_mut() = Rc::downgrade(&branch);

        println!(
            "branch strong = {}, weak = {}",
            Rc::strong_count(&branch),
            Rc::weak_count(&branch),
        );

        println!(
            "leaf strong = {}, weak = {}",
            Rc::strong_count(&leaf),
            Rc::weak_count(&leaf),
        );
    }

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());
    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );
}
"},{"title":"rust functional programming","date":"2023-04-11T14:04:38.000Z","_content":"\n## iterator\n- `iter()` Returns an iterator over the **slice**\n- `into_iter()` Creates a consuming iterator, that is, one that moves each value out of the vector\n- `iter_mut()` Returns an iterator that allows modifying each value.\n\n## flat_map\n```rust\n/// Creates an iterator that works like map, but flattens nested structure.\n///\n/// The [`map`] adapter is very useful, but only when the closure\n/// argument produces values. If it produces an iterator instead, there's\n/// an extra layer of indirection. `flat_map()` will remove this extra layer\n/// on its own.\n```\n### Examples\n```rust\nlet words = [\"alpha\", \"beta\", \"gamma\"];\n// chars() returns an iterator\nlet merged: String = words.iter()\n .flat_map(|s| s.chars())\n .collect();\nassert_eq!(merged, \"alphabetagamma\");\n```\n","source":"_posts/rust/rust-09-functional.md","raw":"---\ntitle: rust functional programming\ndate: 2023-04-11 22:04:38\ntags: [rust]\n---\n\n## iterator\n- `iter()` Returns an iterator over the **slice**\n- `into_iter()` Creates a consuming iterator, that is, one that moves each value out of the vector\n- `iter_mut()` Returns an iterator that allows modifying each value.\n\n## flat_map\n```rust\n/// Creates an iterator that works like map, but flattens nested structure.\n///\n/// The [`map`] adapter is very useful, but only when the closure\n/// argument produces values. If it produces an iterator instead, there's\n/// an extra layer of indirection. `flat_map()` will remove this extra layer\n/// on its own.\n```\n### Examples\n```rust\nlet words = [\"alpha\", \"beta\", \"gamma\"];\n// chars() returns an iterator\nlet merged: String = words.iter()\n .flat_map(|s| s.chars())\n .collect();\nassert_eq!(merged, \"alphabetagamma\");\n```\n","slug":"rust/rust-09-functional","published":1,"updated":"2023-06-29T01:53:41.688Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzq0012ofsjcu8w4p8b","content":"

iterator

    \n
  • iter() Returns an iterator over the slice
    • \n
  • into_iter() Creates a consuming iterator, that is, one that moves each value out of the vector
    • \n
  • iter_mut() Returns an iterator that allows modifying each value.
    • \n
\n

flat_map

1
2
3
4
5
6
/// Creates an iterator that works like map, but flattens nested structure.
///
/// The [`map`] adapter is very useful, but only when the closure
/// argument produces values. If it produces an iterator instead, there's
/// an extra layer of indirection. `flat_map()` will remove this extra layer
/// on its own.
\n

Examples

1
2
3
4
5
6
let words = ["alpha", "beta", "gamma"];
// chars() returns an iterator
let merged: String = words.iter()
                          .flat_map(|s| s.chars())
                          .collect();
assert_eq!(merged, "alphabetagamma");
\n","site":{"data":{}},"excerpt":"","more":"

iterator

    \n
  • iter() Returns an iterator over the slice
    • \n
  • into_iter() Creates a consuming iterator, that is, one that moves each value out of the vector
    • \n
  • iter_mut() Returns an iterator that allows modifying each value.
    • \n
\n

flat_map

1
2
3
4
5
6
/// Creates an iterator that works like map, but flattens nested structure.
///
/// The [`map`] adapter is very useful, but only when the closure
/// argument produces values. If it produces an iterator instead, there's
/// an extra layer of indirection. `flat_map()` will remove this extra layer
/// on its own.
\n

Examples

1
2
3
4
5
6
let words = ["alpha", "beta", "gamma"];
// chars() returns an iterator
let merged: String = words.iter()
                          .flat_map(|s| s.chars())
                          .collect();
assert_eq!(merged, "alphabetagamma");
\n"},{"title":"rust concurrency","date":"2023-06-01T14:04:38.000Z","_content":"\n## Send and Sync\n- A type is Send if it is safe to send it to another thread.\n- A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).\n- raw pointers are neither Send nor Sync (because they have no safety guards).\n- UnsafeCell isn't Sync (and therefore Cell and RefCell aren't).\n- Rc isn't Send or Sync (because the refcount is shared and unsynchronized).\n\nTypes that aren't automatically derived can simply implement them if desired:\n```rust\nstruct MyBox(*mut u8);\n\nunsafe impl Send for MyBox {}\nunsafe impl Sync for MyBox {}\n```\none can also unimplement Send and Sync:\n```rust\n#![feature(negative_impls)]\n\n// I have some magic semantics for some synchronization primitive!\nstruct SpecialThreadToken(u8);\n\nimpl !Send for SpecialThreadToken {}\nimpl !Sync for SpecialThreadToken {}\n```\n\n## reference\n- [rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html)","source":"_posts/rust/rust-10-concurrency.md","raw":"---\ntitle: rust concurrency\ndate: 2023-06-01 22:04:38\ntags: [rust]\n---\n\n## Send and Sync\n- A type is Send if it is safe to send it to another thread.\n- A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).\n- raw pointers are neither Send nor Sync (because they have no safety guards).\n- UnsafeCell isn't Sync (and therefore Cell and RefCell aren't).\n- Rc isn't Send or Sync (because the refcount is shared and unsynchronized).\n\nTypes that aren't automatically derived can simply implement them if desired:\n```rust\nstruct MyBox(*mut u8);\n\nunsafe impl Send for MyBox {}\nunsafe impl Sync for MyBox {}\n```\none can also unimplement Send and Sync:\n```rust\n#![feature(negative_impls)]\n\n// I have some magic semantics for some synchronization primitive!\nstruct SpecialThreadToken(u8);\n\nimpl !Send for SpecialThreadToken {}\nimpl !Sync for SpecialThreadToken {}\n```\n\n## reference\n- [rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html)","slug":"rust/rust-10-concurrency","published":1,"updated":"2023-07-02T07:42:23.897Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzr0013ofsj6wi71v7f","content":"

Send and Sync

    \n
  • A type is Send if it is safe to send it to another thread.
    • \n
  • A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).
    • \n
  • raw pointers are neither Send nor Sync (because they have no safety guards).
    • \n
  • UnsafeCell isn’t Sync (and therefore Cell and RefCell aren’t).
    • \n
  • Rc isn’t Send or Sync (because the refcount is shared and unsynchronized).
    • \n
\n

Types that aren’t automatically derived can simply implement them if desired:

\n
1
2
3
4
struct MyBox(*mut u8);

unsafe impl Send for MyBox {}
unsafe impl Sync for MyBox {}
\n

one can also unimplement Send and Sync:

\n
1
2
3
4
5
6
7
#![feature(negative_impls)]

// I have some magic semantics for some synchronization primitive!
struct SpecialThreadToken(u8);

impl !Send for SpecialThreadToken {}
impl !Sync for SpecialThreadToken {}
\n\n

reference

\n","site":{"data":{}},"excerpt":"","more":"

Send and Sync

    \n
  • A type is Send if it is safe to send it to another thread.
    • \n
  • A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).
    • \n
  • raw pointers are neither Send nor Sync (because they have no safety guards).
    • \n
  • UnsafeCell isn’t Sync (and therefore Cell and RefCell aren’t).
    • \n
  • Rc isn’t Send or Sync (because the refcount is shared and unsynchronized).
    • \n
\n

Types that aren’t automatically derived can simply implement them if desired:

\n
1
2
3
4
struct MyBox(*mut u8);

unsafe impl Send for MyBox {}
unsafe impl Sync for MyBox {}
\n

one can also unimplement Send and Sync:

\n
1
2
3
4
5
6
7
#![feature(negative_impls)]

// I have some magic semantics for some synchronization primitive!
struct SpecialThreadToken(u8);

impl !Send for SpecialThreadToken {}
impl !Sync for SpecialThreadToken {}
\n\n

reference

\n"},{"title":"rust cargo all in one","date":"2022-12-06T09:05:07.000Z","_content":"\n## useful cmd\n```\ncargo new ${crate_name} --lib ## create a lib crate\ncargo build --verbose ## print out each rustc invocation\n```\n\n## specifying dependencies\n- specifying dependencies from crates.io\n```toml\n[dependencies]\ntime = \"0.1.12\"\n```\n- specifying dependencies from other registries\n```toml\n[dependencies]\nsome-crate = { version = \"1.0\", registry = \"my-registry\" }\n```\n- specifying dependencies form git repositories\n```toml\n[dependencies]\nregex = { git = \"https://github.com/rust-lang/regex.git\" }\n```\n- path dependencies\n```toml\n[dependencies]\nhello_utils = { path = \"hello_utils\" }\n```\n- platform specific dependencies\n```toml\n[target.'cfg(unix)'.dependencies]\nopenssl = \"1.0.1\"\n\n[target.'cfg(target_arch = \"x86\")'.dependencies]\nnative-i686 = { path = \"native/i686\" }\n```\nLike with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.\nIf you want to know which cfg targets are available on your platform, run ```rustc --print=cfg``` from the command line.\nIf you want to know which cfg targets are available for another platform, such as 64-bit Windows, run ```rustc --print=cfg --target=x86_64-pc-windows-msvc```\n- custom target specifications\n```toml\n[target.bar.dependencies]\nwinhttp = \"0.4.0\"\n\n[target.my-special-i686-platform.dependencies]\nopenssl = \"1.0.1\"\nnative = { path = \"native/i686\" }\n```\n- development dependencies\n [dev-dependencies]\n Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.\n These dependencies are not propagated to other packages which depend on this package.\n ```toml\n [target.'cfg(unix)'.dev-dependencies]\n mio = \"0.0.1\"\n ```\n\n- build dependencies\n ```toml\n [build-dependencies]\n cc = \"1.0.3\"\n ```\n The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.\n- choosing features\n ```toml\n [dependencies.awesome]\n version = \"1.3.5\"\n default-features = false # do not include the default features, and optionally\n # cherry-pick individual features\n features = [\"secure-password\", \"civet\"]\n ```\n- renaming dependencies in Cargo.toml\n When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it's published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.\n (more to be found in original book)\n\n## references\n[cargo book](https://doc.rust-lang.org/cargo/)","source":"_posts/rust/rust-cargo-all-in-one.md","raw":"---\ntitle: rust cargo all in one\ndate: 2022-12-06 17:05:07\ntags: [rust]\n---\n\n## useful cmd\n```\ncargo new ${crate_name} --lib ## create a lib crate\ncargo build --verbose ## print out each rustc invocation\n```\n\n## specifying dependencies\n- specifying dependencies from crates.io\n```toml\n[dependencies]\ntime = \"0.1.12\"\n```\n- specifying dependencies from other registries\n```toml\n[dependencies]\nsome-crate = { version = \"1.0\", registry = \"my-registry\" }\n```\n- specifying dependencies form git repositories\n```toml\n[dependencies]\nregex = { git = \"https://github.com/rust-lang/regex.git\" }\n```\n- path dependencies\n```toml\n[dependencies]\nhello_utils = { path = \"hello_utils\" }\n```\n- platform specific dependencies\n```toml\n[target.'cfg(unix)'.dependencies]\nopenssl = \"1.0.1\"\n\n[target.'cfg(target_arch = \"x86\")'.dependencies]\nnative-i686 = { path = \"native/i686\" }\n```\nLike with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.\nIf you want to know which cfg targets are available on your platform, run ```rustc --print=cfg``` from the command line.\nIf you want to know which cfg targets are available for another platform, such as 64-bit Windows, run ```rustc --print=cfg --target=x86_64-pc-windows-msvc```\n- custom target specifications\n```toml\n[target.bar.dependencies]\nwinhttp = \"0.4.0\"\n\n[target.my-special-i686-platform.dependencies]\nopenssl = \"1.0.1\"\nnative = { path = \"native/i686\" }\n```\n- development dependencies\n [dev-dependencies]\n Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.\n These dependencies are not propagated to other packages which depend on this package.\n ```toml\n [target.'cfg(unix)'.dev-dependencies]\n mio = \"0.0.1\"\n ```\n\n- build dependencies\n ```toml\n [build-dependencies]\n cc = \"1.0.3\"\n ```\n The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.\n- choosing features\n ```toml\n [dependencies.awesome]\n version = \"1.3.5\"\n default-features = false # do not include the default features, and optionally\n # cherry-pick individual features\n features = [\"secure-password\", \"civet\"]\n ```\n- renaming dependencies in Cargo.toml\n When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it's published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.\n (more to be found in original book)\n\n## references\n[cargo book](https://doc.rust-lang.org/cargo/)","slug":"rust/rust-cargo-all-in-one","published":1,"updated":"2023-05-03T10:04:18.682Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzr0015ofsjfviv9rjn","content":"

useful cmd

1
2
cargo new ${crate_name} --lib ## create a lib crate
cargo build --verbose ## print out each rustc invocation
\n\n

specifying dependencies

    \n

  • specifying dependencies from crates.io

    \n
    1
    2
    [dependencies]
    time = "0.1.12"
    • \n

  • specifying dependencies from other registries

    \n
    1
    2
    [dependencies]
    some-crate = { version = "1.0", registry = "my-registry" }
    • \n

  • specifying dependencies form git repositories

    \n
    1
    2
    [dependencies]
    regex = { git = "https://github.com/rust-lang/regex.git" }
    • \n

  • path dependencies

    \n
    1
    2
    [dependencies]
    hello_utils = { path = "hello_utils" }
    • \n

  • platform specific dependencies

    \n
    1
    2
    3
    4
    5
    [target.'cfg(unix)'.dependencies]
    openssl = "1.0.1"

    [target.'cfg(target_arch = "x86")'.dependencies]
    native-i686 = { path = "native/i686" }
    \n

    Like with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.
    If you want to know which cfg targets are available on your platform, run rustc --print=cfg from the command line.
    If you want to know which cfg targets are available for another platform, such as 64-bit Windows, run rustc --print=cfg --target=x86_64-pc-windows-msvc

    \n
    • \n

  • custom target specifications

    \n
    1
    2
    3
    4
    5
    6
    [target.bar.dependencies]
    winhttp = "0.4.0"

    [target.my-special-i686-platform.dependencies]
    openssl = "1.0.1"
    native = { path = "native/i686" }
    • \n

  • development dependencies
    [dev-dependencies]
    Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.
    These dependencies are not propagated to other packages which depend on this package.

    \n 
    1
    2
    [target.'cfg(unix)'.dev-dependencies]
    mio = "0.0.1"
    \n
    • \n

  • build dependencies

    \n 
    1
    2
    [build-dependencies]
    cc = "1.0.3"
    \n

    The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.

    \n
    • \n

  • choosing features

    \n 
    1
    2
    3
    4
    5
    [dependencies.awesome]
    version = "1.3.5"
    default-features = false # do not include the default features, and optionally
                            # cherry-pick individual features
    features = ["secure-password", "civet"]
    • \n

  • renaming dependencies in Cargo.toml
    When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it’s published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.
    (more to be found in original book)

    \n
    • \n
\n

references

cargo book

\n","site":{"data":{}},"excerpt":"","more":"

useful cmd

1
2
cargo new ${crate_name} --lib ## create a lib crate
cargo build --verbose ## print out each rustc invocation
\n\n

specifying dependencies

    \n

  • specifying dependencies from crates.io

    \n
    1
    2
    [dependencies]
    time = "0.1.12"
    • \n

  • specifying dependencies from other registries

    \n
    1
    2
    [dependencies]
    some-crate = { version = "1.0", registry = "my-registry" }
    • \n

  • specifying dependencies form git repositories

    \n
    1
    2
    [dependencies]
    regex = { git = "https://github.com/rust-lang/regex.git" }
    • \n

  • path dependencies

    \n
    1
    2
    [dependencies]
    hello_utils = { path = "hello_utils" }
    • \n

  • platform specific dependencies

    \n
    1
    2
    3
    4
    5
    [target.'cfg(unix)'.dependencies]
    openssl = "1.0.1"

    [target.'cfg(target_arch = "x86")'.dependencies]
    native-i686 = { path = "native/i686" }
    \n

    Like with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.
    If you want to know which cfg targets are available on your platform, run rustc --print=cfg from the command line.
    If you want to know which cfg targets are available for another platform, such as 64-bit Windows, run rustc --print=cfg --target=x86_64-pc-windows-msvc

    \n
    • \n

  • custom target specifications

    \n
    1
    2
    3
    4
    5
    6
    [target.bar.dependencies]
    winhttp = "0.4.0"

    [target.my-special-i686-platform.dependencies]
    openssl = "1.0.1"
    native = { path = "native/i686" }
    • \n

  • development dependencies
    [dev-dependencies]
    Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.
    These dependencies are not propagated to other packages which depend on this package.

    \n 
    1
    2
    [target.'cfg(unix)'.dev-dependencies]
    mio = "0.0.1"
    \n
    • \n

  • build dependencies

    \n 
    1
    2
    [build-dependencies]
    cc = "1.0.3"
    \n

    The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.

    \n
    • \n

  • choosing features

    \n 
    1
    2
    3
    4
    5
    [dependencies.awesome]
    version = "1.3.5"
    default-features = false # do not include the default features, and optionally
                            # cherry-pick individual features
    features = ["secure-password", "civet"]
    • \n

  • renaming dependencies in Cargo.toml
    When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it’s published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.
    (more to be found in original book)

    \n
    • \n
\n

references

cargo book

\n"},{"title":"rust async","date":"2023-01-13T09:17:10.000Z","_content":" \n\n## I/O\nI/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口\n\n\n### I/O 模型\n```plantuml\n@startmindmap\n+ **I/O模型**\n++ 同步I/O\n'tag::details[]\n+++_ 阻塞I/O (BIO)\n+++_ 非阻塞I/O (NIO)\n+++_ I/O多路复用\n+++_ 信号驱动I/O\n'end::details[]\n++ 异步I/O\n'tag::details[]\n+++_ linux (AIO, io_uring)\n+++_ windows (IOCP)\n'end::details[]\n@endmindmap\n```\n#### 同步阻塞\n- 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；\n- 此时用户线程阻塞，等待内核将数据准备好；\n- 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。\n\n#### 同步非阻塞\n- 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；\n- 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；\n- 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。\n\n#### 同步多路复用\n- 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）\n- 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；\n\n#### 异步I/O\n- 用户线程进行aio_read，进行系统调用切换到内核；\n- 内核立即返回，并不会阻塞用户线程；\n- 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。\n\n### 流程图\n#### 同步blocking I/O\n\n```plantuml\n@startuml Test Diagram\n\nparticipant \"application\" as app\nparticipant \"kernel\" as kernel\n\nactivate app\nactivate kernel\napp -> kernel: syscall: Read recvfrom\nkernel -> kernel: wait for data (no datagram ready)\nkernel -> kernel: copy datagram to user (datagram ready)\nkernel -> app: return\n@enduml\n```\n\n#### I/O多路复用\n\n### 异步编程\n\n\n## the Future Trait\nA Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this\n\n```rust\n\ntrait SimpleFuture {\n type Output;\n fn poll(&mut self, wake: fn()) -> Poll;\n}\n\nenum Poll {\n Ready(T),\n Pending,\n}\n```\n\nFor example, consider the case where we want to read from a socket that may or may not have data available already.\n```rust\npub struct SocketRead<'a> {\n socket: &'a Socket,\n}\n\nimpl SimpleFuture for SocketRead<'_> {\n type Output = Vec;\n\n fn poll(&mut self, wake: fn()) -> Poll {\n if self.socket.has_data_to_read() {\n // The socket has data -- read it into a buffer and return it.\n Poll::Ready(self.socket.read_buf())\n } else {\n // The socket does not yet have data.\n //\n // Arrange for `wake` to be called once data is available.\n // When data becomes available, `wake` will be called, and the\n // user of this `Future` will know to call `poll` again and\n // receive data.\n self.socket.set_readable_callback(wake);\n Poll::Pending\n }\n }\n}\n\n```\n\n the real Future trait and how it is different\n```rust\ntrait Future {\n type Output;\n fn poll(\n // Note the change from `&mut self` to `Pin<&mut Self>`:\n self: Pin<&mut Self>,\n // and the change from `wake: fn()` to `cx: &mut Context<'_>`:\n cx: &mut Context<'_>,\n ) -> Poll;\n}\n\n```\nThe first change you'll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We'll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.\n\nSecondly, wake: fn() has changed to &mut Context<'_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can't store any data about which Future called wake.\n\nIn a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.\n\n\n## task wakeups with Waker\nWaker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.\n\n## referencs\n[csdn blog](https://blog.csdn.net/XMJYever/article/details/111560976)\n","source":"_posts/rust/rust-async.md","raw":"---\ntitle: rust async\ndate: 2023-01-13 17:17:10\ntags: [rust]\n--- \n\n## I/O\nI/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口\n\n\n### I/O 模型\n```plantuml\n@startmindmap\n+ **I/O模型**\n++ 同步I/O\n'tag::details[]\n+++_ 阻塞I/O (BIO)\n+++_ 非阻塞I/O (NIO)\n+++_ I/O多路复用\n+++_ 信号驱动I/O\n'end::details[]\n++ 异步I/O\n'tag::details[]\n+++_ linux (AIO, io_uring)\n+++_ windows (IOCP)\n'end::details[]\n@endmindmap\n```\n#### 同步阻塞\n- 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；\n- 此时用户线程阻塞，等待内核将数据准备好；\n- 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。\n\n#### 同步非阻塞\n- 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；\n- 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；\n- 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。\n\n#### 同步多路复用\n- 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）\n- 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；\n\n#### 异步I/O\n- 用户线程进行aio_read，进行系统调用切换到内核；\n- 内核立即返回，并不会阻塞用户线程；\n- 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。\n\n### 流程图\n#### 同步blocking I/O\n\n```plantuml\n@startuml Test Diagram\n\nparticipant \"application\" as app\nparticipant \"kernel\" as kernel\n\nactivate app\nactivate kernel\napp -> kernel: syscall: Read recvfrom\nkernel -> kernel: wait for data (no datagram ready)\nkernel -> kernel: copy datagram to user (datagram ready)\nkernel -> app: return\n@enduml\n```\n\n#### I/O多路复用\n\n### 异步编程\n\n\n## the Future Trait\nA Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this\n\n```rust\n\ntrait SimpleFuture {\n type Output;\n fn poll(&mut self, wake: fn()) -> Poll;\n}\n\nenum Poll {\n Ready(T),\n Pending,\n}\n```\n\nFor example, consider the case where we want to read from a socket that may or may not have data available already.\n```rust\npub struct SocketRead<'a> {\n socket: &'a Socket,\n}\n\nimpl SimpleFuture for SocketRead<'_> {\n type Output = Vec;\n\n fn poll(&mut self, wake: fn()) -> Poll {\n if self.socket.has_data_to_read() {\n // The socket has data -- read it into a buffer and return it.\n Poll::Ready(self.socket.read_buf())\n } else {\n // The socket does not yet have data.\n //\n // Arrange for `wake` to be called once data is available.\n // When data becomes available, `wake` will be called, and the\n // user of this `Future` will know to call `poll` again and\n // receive data.\n self.socket.set_readable_callback(wake);\n Poll::Pending\n }\n }\n}\n\n```\n\n the real Future trait and how it is different\n```rust\ntrait Future {\n type Output;\n fn poll(\n // Note the change from `&mut self` to `Pin<&mut Self>`:\n self: Pin<&mut Self>,\n // and the change from `wake: fn()` to `cx: &mut Context<'_>`:\n cx: &mut Context<'_>,\n ) -> Poll;\n}\n\n```\nThe first change you'll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We'll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.\n\nSecondly, wake: fn() has changed to &mut Context<'_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can't store any data about which Future called wake.\n\nIn a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.\n\n\n## task wakeups with Waker\nWaker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.\n\n## referencs\n[csdn blog](https://blog.csdn.net/XMJYever/article/details/111560976)\n","slug":"rust/rust-async","published":1,"updated":"2023-05-03T09:33:13.753Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzr0016ofsjdqxn9hhz","content":"

I/O

I/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口

\n

I/O 模型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
@startmindmap
+ **I/O模型**
++ 同步I/O
'tag::details[]
+++_ 阻塞I/O (BIO)
+++_ 非阻塞I/O (NIO)
+++_ I/O多路复用
+++_ 信号驱动I/O
'end::details[]
++ 异步I/O
'tag::details[]
+++_ linux (AIO, io_uring)
+++_ windows (IOCP)
'end::details[]
@endmindmap
\n

同步阻塞

    \n
  • 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；
    • \n
  • 此时用户线程阻塞，等待内核将数据准备好；
    • \n
  • 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。
    • \n
\n

同步非阻塞

    \n
  • 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；
    • \n
  • 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；
    • \n
  • 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。
    • \n
\n

同步多路复用

    \n
  • 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）
    • \n
  • 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；
    • \n
\n

异步I/O

    \n
  • 用户线程进行aio_read，进行系统调用切换到内核；
    • \n
  • 内核立即返回，并不会阻塞用户线程；
    • \n
  • 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。
    • \n
\n

流程图

同步blocking I/O

1
2
3
4
5
6
7
8
9
10
11
12
@startuml Test Diagram

participant "application" as app
participant "kernel" as kernel

activate app
activate kernel
app -> kernel: syscall: Read recvfrom
kernel -> kernel: wait for data (no datagram ready)
kernel -> kernel: copy datagram to user (datagram ready)
kernel -> app: return
@enduml
\n\n

I/O多路复用

异步编程

the Future Trait

A Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this

\n
1
2
3
4
5
6
7
8
9
10

trait SimpleFuture {
    type Output;
    fn poll(&mut self, wake: fn()) -> Poll<Self::Output>;
}

enum Poll<T> {
    Ready(T),
    Pending,
}
\n\n

For example, consider the case where we want to read from a socket that may or may not have data available already.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
pub struct SocketRead<'a> {
    socket: &'a Socket,
}

impl SimpleFuture for SocketRead<'_> {
    type Output = Vec<u8>;

    fn poll(&mut self, wake: fn()) -> Poll<Self::Output> {
        if self.socket.has_data_to_read() {
            // The socket has data -- read it into a buffer and return it.
            Poll::Ready(self.socket.read_buf())
        } else {
            // The socket does not yet have data.
            //
            // Arrange for `wake` to be called once data is available.
            // When data becomes available, `wake` will be called, and the
            // user of this `Future` will know to call `poll` again and
            // receive data.
            self.socket.set_readable_callback(wake);
            Poll::Pending
        }
    }
}

\n\n

the real Future trait and how it is different

\n
1
2
3
4
5
6
7
8
9
10
trait Future {
    type Output;
    fn poll(
        // Note the change from `&mut self` to `Pin<&mut Self>`:
        self: Pin<&mut Self>,
        // and the change from `wake: fn()` to `cx: &mut Context<'_>`:
        cx: &mut Context<'_>,
    ) -> Poll<Self::Output>;
}

\n

The first change you’ll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We’ll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.

\n

Secondly, wake: fn() has changed to &mut Context<’_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can’t store any data about which Future called wake.

\n

In a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.

\n

task wakeups with Waker

Waker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.

\n

referencs

csdn blog

\n","site":{"data":{}},"excerpt":"","more":"

I/O

I/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口

\n

I/O 模型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
@startmindmap
+ **I/O模型**
++ 同步I/O
'tag::details[]
+++_ 阻塞I/O (BIO)
+++_ 非阻塞I/O (NIO)
+++_ I/O多路复用
+++_ 信号驱动I/O
'end::details[]
++ 异步I/O
'tag::details[]
+++_ linux (AIO, io_uring)
+++_ windows (IOCP)
'end::details[]
@endmindmap
\n

同步阻塞

    \n
  • 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；
    • \n
  • 此时用户线程阻塞，等待内核将数据准备好；
    • \n
  • 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。
    • \n
\n

同步非阻塞

    \n
  • 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；
    • \n
  • 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；
    • \n
  • 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。
    • \n
\n

同步多路复用

    \n
  • 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）
    • \n
  • 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；
    • \n
\n

异步I/O

    \n
  • 用户线程进行aio_read，进行系统调用切换到内核；
    • \n
  • 内核立即返回，并不会阻塞用户线程；
    • \n
  • 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。
    • \n
\n

流程图

同步blocking I/O

1
2
3
4
5
6
7
8
9
10
11
12
@startuml Test Diagram

participant "application" as app
participant "kernel" as kernel

activate app
activate kernel
app -> kernel: syscall: Read recvfrom
kernel -> kernel: wait for data (no datagram ready)
kernel -> kernel: copy datagram to user (datagram ready)
kernel -> app: return
@enduml
\n\n

I/O多路复用

异步编程

the Future Trait

A Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this

\n
1
2
3
4
5
6
7
8
9
10

trait SimpleFuture {
    type Output;
    fn poll(&mut self, wake: fn()) -> Poll<Self::Output>;
}

enum Poll<T> {
    Ready(T),
    Pending,
}
\n\n

For example, consider the case where we want to read from a socket that may or may not have data available already.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
pub struct SocketRead<'a> {
    socket: &'a Socket,
}

impl SimpleFuture for SocketRead<'_> {
    type Output = Vec<u8>;

    fn poll(&mut self, wake: fn()) -> Poll<Self::Output> {
        if self.socket.has_data_to_read() {
            // The socket has data -- read it into a buffer and return it.
            Poll::Ready(self.socket.read_buf())
        } else {
            // The socket does not yet have data.
            //
            // Arrange for `wake` to be called once data is available.
            // When data becomes available, `wake` will be called, and the
            // user of this `Future` will know to call `poll` again and
            // receive data.
            self.socket.set_readable_callback(wake);
            Poll::Pending
        }
    }
}

\n\n

the real Future trait and how it is different

\n
1
2
3
4
5
6
7
8
9
10
trait Future {
    type Output;
    fn poll(
        // Note the change from `&mut self` to `Pin<&mut Self>`:
        self: Pin<&mut Self>,
        // and the change from `wake: fn()` to `cx: &mut Context<'_>`:
        cx: &mut Context<'_>,
    ) -> Poll<Self::Output>;
}

\n

The first change you’ll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We’ll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.

\n

Secondly, wake: fn() has changed to &mut Context<’_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can’t store any data about which Future called wake.

\n

In a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.

\n

task wakeups with Waker

Waker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.

\n

referencs

csdn blog

\n"},{"title":"cargo doc","date":"2022-11-13T07:41:59.000Z","_content":"\n## 文档注释\n### 用于生成文档\n - 使用 ///\n - 支持 Markdown\n - 放置在被说没条目之前\n### 例子\n```rust\n/// adds one to the number given\n/// \n/// # Examples\n/// ```\n/// let arg = 5;\n/// let answer = my_crate::add_one(arg);\n/// \n/// assert_eq!(6, answer);\n/// ```\npub fn add_one(x: i32) -> i32 {\n x + 1;\n}\n```\n### 命令\n```\ncargo doc ## 生成的html放在 target/doc 目录下\ncargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)\n```\n### 常用章节\n- `# Examples`\n- `Panics`: 可能panic的场景\n- `Errors`: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件\n- `Safety`: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提\n\n### 文档注释作为测试\n- 运行cargo test, doc中用# Example标记的实例代码会用来测试运行\n\n### 为包含注释的项添加文档注释\n- 符号: //!\n- 这类注释通常用描述crate和模块:\n - crate root (按惯例 src/lib.rs)\n - 一个模块内，将crate火模块作为一个整体进行记录\n\n## 注释\n//! - 模块级稳定注释, 置于模块头部\n//!! - 模块级稳定注释, 但是和上面注释置于同一行\n\n//! - 模块级稳定注释, 会换行\n\n/*! - 模块级稳定注释 */\n/*!！ - 模块级稳定注释, 和上面同一行 */\n\n// 普通行注释\n/// 行级文档注释\n//// 普通行注释\n\n/* 普通块级注释 */\n/** 会级文档注释 */","source":"_posts/rust/rust-cargo-doc.md","raw":"---\ntitle: cargo doc\ndate: 2022-11-13 15:41:59\ntags: [rust,cargo]\n---\n\n## 文档注释\n### 用于生成文档\n - 使用 ///\n - 支持 Markdown\n - 放置在被说没条目之前\n### 例子\n```rust\n/// adds one to the number given\n/// \n/// # Examples\n/// ```\n/// let arg = 5;\n/// let answer = my_crate::add_one(arg);\n/// \n/// assert_eq!(6, answer);\n/// ```\npub fn add_one(x: i32) -> i32 {\n x + 1;\n}\n```\n### 命令\n```\ncargo doc ## 生成的html放在 target/doc 目录下\ncargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)\n```\n### 常用章节\n- `# Examples`\n- `Panics`: 可能panic的场景\n- `Errors`: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件\n- `Safety`: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提\n\n### 文档注释作为测试\n- 运行cargo test, doc中用# Example标记的实例代码会用来测试运行\n\n### 为包含注释的项添加文档注释\n- 符号: //!\n- 这类注释通常用描述crate和模块:\n - crate root (按惯例 src/lib.rs)\n - 一个模块内，将crate火模块作为一个整体进行记录\n\n## 注释\n//! - 模块级稳定注释, 置于模块头部\n//!! - 模块级稳定注释, 但是和上面注释置于同一行\n\n//! - 模块级稳定注释, 会换行\n\n/*! - 模块级稳定注释 */\n/*!！ - 模块级稳定注释, 和上面同一行 */\n\n// 普通行注释\n/// 行级文档注释\n//// 普通行注释\n\n/* 普通块级注释 */\n/** 会级文档注释 */","slug":"rust/rust-cargo-doc","published":1,"updated":"2023-05-03T09:28:51.353Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzr0018ofsj3y1gc8vv","content":"

文档注释

用于生成文档

    \n
  • 使用 ///
    • \n
  • 支持 Markdown
    • \n
  • 放置在被说没条目之前
    • \n
\n

例子

1
2
3
4
5
6
7
8
9
10
11
12
/// adds one to the number given
/// 
/// # Examples
/// ```
/// let arg = 5;
/// let answer = my_crate::add_one(arg);
/// 
/// assert_eq!(6, answer);
/// ```
pub fn add_one(x: i32) -> i32 {
    x + 1;
}
\n

命令

1
2
cargo doc  ## 生成的html放在 target/doc 目录下
cargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)
\n

常用章节

    \n
  • # Examples
    • \n
  • Panics: 可能panic的场景
    • \n
  • Errors: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件
    • \n
  • Safety: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提
    • \n
\n

文档注释作为测试

    \n
  • 运行cargo test, doc中用# Example标记的实例代码会用来测试运行
    • \n
\n

为包含注释的项添加文档注释

    \n
  • 符号: //!
    • \n
  • 这类注释通常用描述crate和模块:
      \n
    • crate root (按惯例 src/lib.rs)
      • \n
    • 一个模块内，将crate火模块作为一个整体进行记录
      • \n
    \n
    • \n
\n

注释

//! - 模块级稳定注释, 置于模块头部
//!! - 模块级稳定注释, 但是和上面注释置于同一行

\n

//! - 模块级稳定注释, 会换行

\n

/*! - 模块级稳定注释 /
/!！ - 模块级稳定注释, 和上面同一行 */

\n

// 普通行注释
/// 行级文档注释
//// 普通行注释

\n

/* 普通块级注释 /
/* 会级文档注释 */

\n","site":{"data":{}},"excerpt":"","more":"

文档注释

用于生成文档

    \n
  • 使用 ///
    • \n
  • 支持 Markdown
    • \n
  • 放置在被说没条目之前
    • \n
\n

例子

1
2
3
4
5
6
7
8
9
10
11
12
/// adds one to the number given
/// 
/// # Examples
/// ```
/// let arg = 5;
/// let answer = my_crate::add_one(arg);
/// 
/// assert_eq!(6, answer);
/// ```
pub fn add_one(x: i32) -> i32 {
    x + 1;
}
\n

命令

1
2
cargo doc  ## 生成的html放在 target/doc 目录下
cargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)
\n

常用章节

    \n
  • # Examples
    • \n
  • Panics: 可能panic的场景
    • \n
  • Errors: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件
    • \n
  • Safety: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提
    • \n
\n

文档注释作为测试

    \n
  • 运行cargo test, doc中用# Example标记的实例代码会用来测试运行
    • \n
\n

为包含注释的项添加文档注释

    \n
  • 符号: //!
    • \n
  • 这类注释通常用描述crate和模块:
      \n
    • crate root (按惯例 src/lib.rs)
      • \n
    • 一个模块内，将crate火模块作为一个整体进行记录
      • \n
    \n
    • \n
\n

注释

//! - 模块级稳定注释, 置于模块头部
//!! - 模块级稳定注释, 但是和上面注释置于同一行

\n

//! - 模块级稳定注释, 会换行

\n

/*! - 模块级稳定注释 /
/!！ - 模块级稳定注释, 和上面同一行 */

\n

// 普通行注释
/// 行级文档注释
//// 普通行注释

\n

/* 普通块级注释 /
/* 会级文档注释 */

\n"},{"title":"rust similar concepts comparison","date":"2022-11-23T07:52:34.000Z","_content":"\n## ref vs &\n`ref` annotates pattern bindings to make them borrow rather than move. It is **not** a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.\nBy default, match statements consume all they can, which can sometimes be a problem, when you don't really need the value to be moved and owned:\n```rust\nlet maybe_name = Some(String::from(\"Alice\"));\n// Using `ref`, the value is borrowed, not moved ...\nmatch maybe_name {\n Some(ref n) => println!(\"Hello, {n}\"),\n _ => println!(\"Hello, world\"),\n}\n// ... so it's available here!\nprintln!(\"Hello again, {}\", maybe_name.unwrap_or(\"world\".into()));\n```\n\n- `&` denotes that your pattern expects a reference to an object. Hence `&` is a part of said pattern: `&Foo` matches different objects than `Foo` does.\n- `ref` indicates that you want a reference to an unpacked value. It is not matched against: `Foo(ref foo)` matches the same objects as `Foo(foo)`.\n\n## Clone vs Copy\n### Copy 的含义\n`Copy` 的全名是 `std::marker::Copy`。`std::marker` 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 `Copy`、`Send`、`Sized`、`Sync`。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。\n\n如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。\n\n一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。\n\n### Copy 的实现条件\n并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。\n\n常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。\n\n我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。\n\n### Clone 的含义\nClone 的全名是 std::clone::Clone。它的完整声明是这样的：\n```rust\npub trait Clone : Sized {\n fn clone(&self) -> Self;\n fn clone_from(&mut self, source: &Self) {\n *self = source.clone()\n }\n}\n```\n它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。\n\nclone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。\n\n虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。\n\n### 自动 derive\n绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：\n\n```rust\n#[derive(Copy, Clone)]\nstruct MyStruct(i32);\n```\n这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。\n\n通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。\n\n目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。\n\n## Cell vs RefCell\n- Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(\"Hello\")).get()会报错\n- Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic\n- 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大\n\n## AsRef vs Borrow\n[WIP]\n","source":"_posts/rust/rust-similar-concepts-comparison.md","raw":"---\ntitle: rust similar concepts comparison\ndate: 2022-11-23 15:52:34\ntags: [rust]\n---\n\n## ref vs &\n`ref` annotates pattern bindings to make them borrow rather than move. It is **not** a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.\nBy default, match statements consume all they can, which can sometimes be a problem, when you don't really need the value to be moved and owned:\n```rust\nlet maybe_name = Some(String::from(\"Alice\"));\n// Using `ref`, the value is borrowed, not moved ...\nmatch maybe_name {\n Some(ref n) => println!(\"Hello, {n}\"),\n _ => println!(\"Hello, world\"),\n}\n// ... so it's available here!\nprintln!(\"Hello again, {}\", maybe_name.unwrap_or(\"world\".into()));\n```\n\n- `&` denotes that your pattern expects a reference to an object. Hence `&` is a part of said pattern: `&Foo` matches different objects than `Foo` does.\n- `ref` indicates that you want a reference to an unpacked value. It is not matched against: `Foo(ref foo)` matches the same objects as `Foo(foo)`.\n\n## Clone vs Copy\n### Copy 的含义\n`Copy` 的全名是 `std::marker::Copy`。`std::marker` 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 `Copy`、`Send`、`Sized`、`Sync`。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。\n\n如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。\n\n一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。\n\n### Copy 的实现条件\n并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。\n\n常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。\n\n我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。\n\n### Clone 的含义\nClone 的全名是 std::clone::Clone。它的完整声明是这样的：\n```rust\npub trait Clone : Sized {\n fn clone(&self) -> Self;\n fn clone_from(&mut self, source: &Self) {\n *self = source.clone()\n }\n}\n```\n它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。\n\nclone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。\n\n虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。\n\n### 自动 derive\n绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：\n\n```rust\n#[derive(Copy, Clone)]\nstruct MyStruct(i32);\n```\n这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。\n\n通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。\n\n目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。\n\n## Cell vs RefCell\n- Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(\"Hello\")).get()会报错\n- Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic\n- 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大\n\n## AsRef vs Borrow\n[WIP]\n","slug":"rust/rust-similar-concepts-comparison","published":1,"updated":"2023-07-09T08:33:27.383Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzr001aofsj3n9p8vvr","content":"

ref vs &

ref annotates pattern bindings to make them borrow rather than move. It is not a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.
By default, match statements consume all they can, which can sometimes be a problem, when you don’t really need the value to be moved and owned:

\n
1
2
3
4
5
6
7
8
let maybe_name = Some(String::from("Alice"));
// Using `ref`, the value is borrowed, not moved ...
match maybe_name {
    Some(ref n) => println!("Hello, {n}"),
    _ => println!("Hello, world"),
}
// ... so it's available here!
println!("Hello again, {}", maybe_name.unwrap_or("world".into()));
\n\n
    \n
  • & denotes that your pattern expects a reference to an object. Hence & is a part of said pattern: &Foo matches different objects than Foo does.
    • \n
  • ref indicates that you want a reference to an unpacked value. It is not matched against: Foo(ref foo) matches the same objects as Foo(foo).
    • \n
\n

Clone vs Copy

Copy 的含义

Copy 的全名是 std::marker::Copy。std::marker 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 Copy、Send、Sized、Sync。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。

\n

如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。

\n

一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。

\n

Copy 的实现条件

并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。

\n

常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。

\n

我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。

\n

Clone 的含义

Clone 的全名是 std::clone::Clone。它的完整声明是这样的：

\n
1
2
3
4
5
6
pub trait Clone : Sized {
    fn clone(&self) -> Self;
    fn clone_from(&mut self, source: &Self) {
        *self = source.clone()
    }
}
\n

它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。

\n

clone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。

\n

虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。

\n

自动 derive

绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：

\n
1
2
#[derive(Copy, Clone)]
struct MyStruct(i32);
\n

这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。

\n

通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。

\n

目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。

\n

Cell vs RefCell

    \n
  • Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(“Hello”)).get()会报错
    • \n
  • Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic
    • \n
  • 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大
    • \n
\n

AsRef vs Borrow

[WIP]

\n","site":{"data":{}},"excerpt":"","more":"

ref vs &

ref annotates pattern bindings to make them borrow rather than move. It is not a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.
By default, match statements consume all they can, which can sometimes be a problem, when you don’t really need the value to be moved and owned:

\n
1
2
3
4
5
6
7
8
let maybe_name = Some(String::from("Alice"));
// Using `ref`, the value is borrowed, not moved ...
match maybe_name {
    Some(ref n) => println!("Hello, {n}"),
    _ => println!("Hello, world"),
}
// ... so it's available here!
println!("Hello again, {}", maybe_name.unwrap_or("world".into()));
\n\n
    \n
  • & denotes that your pattern expects a reference to an object. Hence & is a part of said pattern: &Foo matches different objects than Foo does.
    • \n
  • ref indicates that you want a reference to an unpacked value. It is not matched against: Foo(ref foo) matches the same objects as Foo(foo).
    • \n
\n

Clone vs Copy

Copy 的含义

Copy 的全名是 std::marker::Copy。std::marker 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 Copy、Send、Sized、Sync。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。

\n

如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。

\n

一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。

\n

Copy 的实现条件

并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。

\n

常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。

\n

我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。

\n

Clone 的含义

Clone 的全名是 std::clone::Clone。它的完整声明是这样的：

\n
1
2
3
4
5
6
pub trait Clone : Sized {
    fn clone(&self) -> Self;
    fn clone_from(&mut self, source: &Self) {
        *self = source.clone()
    }
}
\n

它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。

\n

clone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。

\n

虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。

\n

自动 derive

绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：

\n
1
2
#[derive(Copy, Clone)]
struct MyStruct(i32);
\n

这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。

\n

通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。

\n

目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。

\n

Cell vs RefCell

    \n
  • Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(“Hello”)).get()会报错
    • \n
  • Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic
    • \n
  • 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大
    • \n
\n

AsRef vs Borrow

[WIP]

\n"},{"title":"rust tools","date":"2022-11-20T09:14:05.000Z","_content":"\n## tools\n- cargo-edit\nThis tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line\n\n- cargo whatfeatures ${crate}\n eg: `cargo whatfeatures hyper`\n","source":"_posts/rust/rust-tools.md","raw":"---\ntitle: rust tools\ndate: 2022-11-20 17:14:05\ntags: [rust]\n---\n\n## tools\n- cargo-edit\nThis tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line\n\n- cargo whatfeatures ${crate}\n eg: `cargo whatfeatures hyper`\n","slug":"rust/rust-tools","published":1,"updated":"2023-05-03T09:25:04.042Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzs001dofsj3nu0e5po","content":"

tools

    \n

  • cargo-edit
    This tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line

    \n
    • \n

  • cargo whatfeatures ${crate}
    eg: cargo whatfeatures hyper

    \n
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

tools

    \n

  • cargo-edit
    This tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line

    \n
    • \n

  • cargo whatfeatures ${crate}
    eg: cargo whatfeatures hyper

    \n
    • \n
\n"},{"title":"rust sugar","date":"2022-11-17T07:47:13.000Z","_content":"\n## formatted print\n```rust\n// Positional arguments can be used\nprintln!(\"{0}, this is {1}. {1}, this is {0}\", \"Alice\", \"Bob\");\n// As can named arguments.\nprintln!(\"{subject} {verb} {object}\",\n object=\"the lazy dog\",\n subject=\"the quick brown fox\",\n verb=\"jumps over\");\n// Different formatting can be invoked by specifying the format character\n// after a `:`.\nprintln!(\"Base 10: {}\", 69420); // 69420\nprintln!(\"Base 2 (binary): {:b}\", 69420); // 10000111100101100\nprintln!(\"Base 8 (octal): {:o}\", 69420); // 207454\nprintln!(\"Base 16 (hexadecimal): {:x}\", 69420); // 10f2c\nprintln!(\"Base 16 (hexadecimal): {:X}\", 69420); // 10F2C\n// You can right-justify text with a specified width. This will\n// output \" 1\". (Four white spaces and a \"1\", for a total width of 5.)\nprintln!(\"{number:>5}\", number=1);\n// You can pad numbers with extra zeroes,\n// and left-adjust by flipping the sign. This will output \"10000\".\nprintln!(\"{number:0<5}\", number=1);\n// You can use named arguments in the format specifier by appending a `$`.\nprintln!(\"{number:0>width$}\", number=1, width=5);\n```\n- [reference](https://doc.rust-lang.org/rust-by-example/hello/print.html)\n\n\n## syntax\n```rust\n/// print to stderr\neprintln!(\"server error: {}\", e);\n\nstruct MyTupleStruct(M);\nlet myTupleStruct = MyTupleStruct::(String::from(\"hello\"));\n\nvec.iter().position()\nvec.iter().find()\nvec.iter().any()\n\n\nstatic mut A: u32 = 0;\n```\n\n## attributes\n```rust\n#![allow(warnings)]\n#[allow(dead_code)]\n#![allow(unused)]\n// Suppress all warnings from casts which overflow.\n#![allow(overflowing_literals)]\n#![allow(unreachable_code)]\n```\n\n## memory\nThe compiler will not rearrange the memory layout\n```rust\n#[repr(C)]\nstruct A {\n a:u8,\n b:u32,\n c:u16\n}\n```\n\n## ptr\n```rust\nNonNull::new_unchecked()\n\n#![feature(new_uninit)]\nlet mut five = Box::::new_uninit();\nlet five = unsafe {\n // Deferred initialization:\n five.as_mut_ptr().write(5);\n five.assume_init()\n};\nassert_eq!(*five, 5)\n\nlet zero = Box::::new_zeroed();\nlet zero = unsafe { zero.assume_init() };\nassert_eq!(*zero, 0)\n\nuse std::alloc::{alloc, Layout};\nunsafe {\n let ptr = alloc(Layout::new::()) as *mut i32;\n ptr.write(5);\n let x = Box::from_raw(ptr);\n}\n```","source":"_posts/rust/rust-sugar.md","raw":"---\ntitle: rust sugar\ndate: 2022-11-17 15:47:13\ntags: [rust]\n---\n\n## formatted print\n```rust\n// Positional arguments can be used\nprintln!(\"{0}, this is {1}. {1}, this is {0}\", \"Alice\", \"Bob\");\n// As can named arguments.\nprintln!(\"{subject} {verb} {object}\",\n object=\"the lazy dog\",\n subject=\"the quick brown fox\",\n verb=\"jumps over\");\n// Different formatting can be invoked by specifying the format character\n// after a `:`.\nprintln!(\"Base 10: {}\", 69420); // 69420\nprintln!(\"Base 2 (binary): {:b}\", 69420); // 10000111100101100\nprintln!(\"Base 8 (octal): {:o}\", 69420); // 207454\nprintln!(\"Base 16 (hexadecimal): {:x}\", 69420); // 10f2c\nprintln!(\"Base 16 (hexadecimal): {:X}\", 69420); // 10F2C\n// You can right-justify text with a specified width. This will\n// output \" 1\". (Four white spaces and a \"1\", for a total width of 5.)\nprintln!(\"{number:>5}\", number=1);\n// You can pad numbers with extra zeroes,\n// and left-adjust by flipping the sign. This will output \"10000\".\nprintln!(\"{number:0<5}\", number=1);\n// You can use named arguments in the format specifier by appending a `$`.\nprintln!(\"{number:0>width$}\", number=1, width=5);\n```\n- [reference](https://doc.rust-lang.org/rust-by-example/hello/print.html)\n\n\n## syntax\n```rust\n/// print to stderr\neprintln!(\"server error: {}\", e);\n\nstruct MyTupleStruct(M);\nlet myTupleStruct = MyTupleStruct::(String::from(\"hello\"));\n\nvec.iter().position()\nvec.iter().find()\nvec.iter().any()\n\n\nstatic mut A: u32 = 0;\n```\n\n## attributes\n```rust\n#![allow(warnings)]\n#[allow(dead_code)]\n#![allow(unused)]\n// Suppress all warnings from casts which overflow.\n#![allow(overflowing_literals)]\n#![allow(unreachable_code)]\n```\n\n## memory\nThe compiler will not rearrange the memory layout\n```rust\n#[repr(C)]\nstruct A {\n a:u8,\n b:u32,\n c:u16\n}\n```\n\n## ptr\n```rust\nNonNull::new_unchecked()\n\n#![feature(new_uninit)]\nlet mut five = Box::::new_uninit();\nlet five = unsafe {\n // Deferred initialization:\n five.as_mut_ptr().write(5);\n five.assume_init()\n};\nassert_eq!(*five, 5)\n\nlet zero = Box::::new_zeroed();\nlet zero = unsafe { zero.assume_init() };\nassert_eq!(*zero, 0)\n\nuse std::alloc::{alloc, Layout};\nunsafe {\n let ptr = alloc(Layout::new::()) as *mut i32;\n ptr.write(5);\n let x = Box::from_raw(ptr);\n}\n```","slug":"rust/rust-sugar","published":1,"updated":"2023-07-11T07:36:27.147Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzs001fofsj1g9v6111","content":"

formatted print

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// Positional arguments can be used
println!("{0}, this is {1}. {1}, this is {0}", "Alice", "Bob");
// As can named arguments.
println!("{subject} {verb} {object}",
            object="the lazy dog",
            subject="the quick brown fox",
            verb="jumps over");
// Different formatting can be invoked by specifying the format character
// after a `:`.
println!("Base 10:               {}",   69420); // 69420
println!("Base 2 (binary):       {:b}", 69420); // 10000111100101100
println!("Base 8 (octal):        {:o}", 69420); // 207454
println!("Base 16 (hexadecimal): {:x}", 69420); // 10f2c
println!("Base 16 (hexadecimal): {:X}", 69420); // 10F2C
// You can right-justify text with a specified width. This will
// output "    1". (Four white spaces and a "1", for a total width of 5.)
println!("{number:>5}", number=1);
// You can pad numbers with extra zeroes,
// and left-adjust by flipping the sign. This will output "10000".
println!("{number:0<5}", number=1);
// You can use named arguments in the format specifier by appending a `$`.
println!("{number:0>width$}", number=1, width=5);
\n\n

syntax

1
2
3
4
5
6
7
8
9
10
11
12
/// print to stderr
eprintln!("server error: {}", e);

struct MyTupleStruct<M>(M);
let myTupleStruct =  MyTupleStruct::<String>(String::from("hello"));

vec.iter().position()
vec.iter().find()
vec.iter().any()


static mut A: u32 = 0;
\n\n

attributes

1
2
3
4
5
6
#![allow(warnings)]
#[allow(dead_code)]
#![allow(unused)]
// Suppress all warnings from casts which overflow.
#![allow(overflowing_literals)]
#![allow(unreachable_code)]
\n\n

memory

The compiler will not rearrange the memory layout

\n
1
2
3
4
5
6
#[repr(C)]
struct A {
    a:u8,
    b:u32,
    c:u16
}
\n\n

ptr

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
NonNull::new_unchecked()

#![feature(new_uninit)]
let mut five = Box::<u32>::new_uninit();
let five = unsafe {
    // Deferred initialization:
    five.as_mut_ptr().write(5);
    five.assume_init()
};
assert_eq!(*five, 5)

let zero = Box::<u32>::new_zeroed();
let zero = unsafe { zero.assume_init() };
assert_eq!(*zero, 0)

use std::alloc::{alloc, Layout};
unsafe {
    let ptr = alloc(Layout::new::<i32>()) as *mut i32;
    ptr.write(5);
    let x = Box::from_raw(ptr);
}
","site":{"data":{}},"excerpt":"","more":"

formatted print

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// Positional arguments can be used
println!("{0}, this is {1}. {1}, this is {0}", "Alice", "Bob");
// As can named arguments.
println!("{subject} {verb} {object}",
            object="the lazy dog",
            subject="the quick brown fox",
            verb="jumps over");
// Different formatting can be invoked by specifying the format character
// after a `:`.
println!("Base 10:               {}",   69420); // 69420
println!("Base 2 (binary):       {:b}", 69420); // 10000111100101100
println!("Base 8 (octal):        {:o}", 69420); // 207454
println!("Base 16 (hexadecimal): {:x}", 69420); // 10f2c
println!("Base 16 (hexadecimal): {:X}", 69420); // 10F2C
// You can right-justify text with a specified width. This will
// output "    1". (Four white spaces and a "1", for a total width of 5.)
println!("{number:>5}", number=1);
// You can pad numbers with extra zeroes,
// and left-adjust by flipping the sign. This will output "10000".
println!("{number:0<5}", number=1);
// You can use named arguments in the format specifier by appending a `$`.
println!("{number:0>width$}", number=1, width=5);
\n\n

syntax

1
2
3
4
5
6
7
8
9
10
11
12
/// print to stderr
eprintln!("server error: {}", e);

struct MyTupleStruct<M>(M);
let myTupleStruct =  MyTupleStruct::<String>(String::from("hello"));

vec.iter().position()
vec.iter().find()
vec.iter().any()


static mut A: u32 = 0;
\n\n

attributes

1
2
3
4
5
6
#![allow(warnings)]
#[allow(dead_code)]
#![allow(unused)]
// Suppress all warnings from casts which overflow.
#![allow(overflowing_literals)]
#![allow(unreachable_code)]
\n\n

memory

The compiler will not rearrange the memory layout

\n
1
2
3
4
5
6
#[repr(C)]
struct A {
    a:u8,
    b:u32,
    c:u16
}
\n\n

ptr

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
NonNull::new_unchecked()

#![feature(new_uninit)]
let mut five = Box::<u32>::new_uninit();
let five = unsafe {
    // Deferred initialization:
    five.as_mut_ptr().write(5);
    five.assume_init()
};
assert_eq!(*five, 5)

let zero = Box::<u32>::new_zeroed();
let zero = unsafe { zero.assume_init() };
assert_eq!(*zero, 0)

use std::alloc::{alloc, Layout};
unsafe {
    let ptr = alloc(Layout::new::<i32>()) as *mut i32;
    ptr.write(5);
    let x = Box::from_raw(ptr);
}
"},{"title":"zkp a brief understanding","date":"2023-06-20T06:29:26.000Z","_content":"\n\n\n## introduction\nzk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.\n\nThe example that we will choose is a simple one: proving that you know the solution to a cubic equation: `x**3 + x + 5 == 35`\n\n## reference\n- [vitalik's blog: qap zero to hero](https://medium.com/@VitalikButerin/quadratic-arithmetic-programs-from-zero-to-hero-f6d558cea649)","source":"_posts/cryptography/zkp/zkp-a-brief-understanding.md","raw":"---\ntitle: zkp a brief understanding\ndate: 2023-06-20 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n## introduction\nzk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.\n\nThe example that we will choose is a simple one: proving that you know the solution to a cubic equation: `x**3 + x + 5 == 35`\n\n## reference\n- [vitalik's blog: qap zero to hero](https://medium.com/@VitalikButerin/quadratic-arithmetic-programs-from-zero-to-hero-f6d558cea649)","slug":"cryptography/zkp/zkp-a-brief-understanding","published":1,"updated":"2023-07-12T14:58:22.472Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzt001iofsj5f219qa6","content":"\n\n\n

introduction

zk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.

\n

The example that we will choose is a simple one: proving that you know the solution to a cubic equation: x**3 + x + 5 == 35

\n

reference

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

zk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.

\n

The example that we will choose is a simple one: proving that you know the solution to a cubic equation: x**3 + x + 5 == 35

\n

reference

\n"},{"title":"rpc","date":"2022-11-08T06:23:08.000Z","_content":"\n\n## overview\npackage rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as 'services'. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.\n\n## methods\n### rpc endpoints (callback)\nMethods that satisfy the following criteria are made available for remote access:\n - method must be exported\n - method returns 0, 1 (response or error) or 2 (response and error) values\n\nThe server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.\n\nAn example server which uses the JSON codec:\n```go\ntype CalculatorService struct {}\n\nfunc (s *CalculatorService) Add(a, b int) int {\n return a + b\n}\n\nfunc (s *CalculatorService) Div(a, b int) (int, error) {\n if b == 0 {\n return 0, errors.New(\"divide by zero\")\n }\n return a/b, nil\n}\n\ncalculator := new(CalculatorService)\nserver := NewServer()\nserver.RegisterName(\"calculator\", calculator)\nl, _ := net.ListenUnix(\"unix\", &net.UnixAddr{Net: \"unix\", Name: \"/tmp/calculator.sock\"})\nserver.ServeListener(l)\n```\n\n### subscriptions\nThe package also supports the publish subscribe pattern through the use of subscriptions.\nA method that is considered eligible for notifications must satisfy the following\ncriteria:\n - method must be exported\n - first method argument type must be context.Context\n - method must have return types (rpc.Subscription, error)\n\nAn example method:\n```go\nfunc (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {\n\t\t...\n\t}\n```\n\n### Reverse Calls\nIn any method handler, an instance of rpc.Client can be accessed through the `ClientFromContext` method. Using this client instance, server-to-client method calls can be performed on the RPC connection.\n\n## server\nto start rpc service, the invoking chain is as below\n```\nnode/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]\n```\n\n### API registration\n","source":"_posts/geth/code_analysis/geth.1.rpc.md","raw":"---\ntitle: rpc\ndate: 2022-11-08 14:23:08\ntags: [blockchain, geth]\n---\n\n\n## overview\npackage rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as 'services'. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.\n\n## methods\n### rpc endpoints (callback)\nMethods that satisfy the following criteria are made available for remote access:\n - method must be exported\n - method returns 0, 1 (response or error) or 2 (response and error) values\n\nThe server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.\n\nAn example server which uses the JSON codec:\n```go\ntype CalculatorService struct {}\n\nfunc (s *CalculatorService) Add(a, b int) int {\n return a + b\n}\n\nfunc (s *CalculatorService) Div(a, b int) (int, error) {\n if b == 0 {\n return 0, errors.New(\"divide by zero\")\n }\n return a/b, nil\n}\n\ncalculator := new(CalculatorService)\nserver := NewServer()\nserver.RegisterName(\"calculator\", calculator)\nl, _ := net.ListenUnix(\"unix\", &net.UnixAddr{Net: \"unix\", Name: \"/tmp/calculator.sock\"})\nserver.ServeListener(l)\n```\n\n### subscriptions\nThe package also supports the publish subscribe pattern through the use of subscriptions.\nA method that is considered eligible for notifications must satisfy the following\ncriteria:\n - method must be exported\n - first method argument type must be context.Context\n - method must have return types (rpc.Subscription, error)\n\nAn example method:\n```go\nfunc (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {\n\t\t...\n\t}\n```\n\n### Reverse Calls\nIn any method handler, an instance of rpc.Client can be accessed through the `ClientFromContext` method. Using this client instance, server-to-client method calls can be performed on the RPC connection.\n\n## server\nto start rpc service, the invoking chain is as below\n```\nnode/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]\n```\n\n### API registration\n","slug":"geth/code_analysis/geth.1.rpc","published":1,"updated":"2023-04-27T16:54:24.069Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzt001kofsjh3ln39ny","content":"

overview

package rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as ‘services’. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.

\n

methods

rpc endpoints (callback)

Methods that satisfy the following criteria are made available for remote access:

\n
    \n
  • method must be exported
    • \n
  • method returns 0, 1 (response or error) or 2 (response and error) values
    • \n
\n

The server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.

\n

An example server which uses the JSON codec:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
type CalculatorService struct {}

func (s *CalculatorService) Add(a, b int) int {
    return a + b
}

func (s *CalculatorService) Div(a, b int) (int, error) {
    if b == 0 {
        return 0, errors.New("divide by zero")
    }
    return a/b, nil
}

calculator := new(CalculatorService)
server := NewServer()
server.RegisterName("calculator", calculator)
l, _ := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: "/tmp/calculator.sock"})
server.ServeListener(l)
\n\n

subscriptions

The package also supports the publish subscribe pattern through the use of subscriptions.
A method that is considered eligible for notifications must satisfy the following
criteria:

\n
    \n
  • method must be exported
    • \n
  • first method argument type must be context.Context
    • \n
  • method must have return types (rpc.Subscription, error)
    • \n
\n

An example method:

\n
1
2
3
func (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {
\t\t...
\t}
\n\n

Reverse Calls

In any method handler, an instance of rpc.Client can be accessed through the ClientFromContext method. Using this client instance, server-to-client method calls can be performed on the RPC connection.

\n

server

to start rpc service, the invoking chain is as below

\n
1
node/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]
\n\n

API registration

","site":{"data":{}},"excerpt":"","more":"

overview

package rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as ‘services’. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.

\n

methods

rpc endpoints (callback)

Methods that satisfy the following criteria are made available for remote access:

\n
    \n
  • method must be exported
    • \n
  • method returns 0, 1 (response or error) or 2 (response and error) values
    • \n
\n

The server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.

\n

An example server which uses the JSON codec:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
type CalculatorService struct {}

func (s *CalculatorService) Add(a, b int) int {
    return a + b
}

func (s *CalculatorService) Div(a, b int) (int, error) {
    if b == 0 {
        return 0, errors.New("divide by zero")
    }
    return a/b, nil
}

calculator := new(CalculatorService)
server := NewServer()
server.RegisterName("calculator", calculator)
l, _ := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: "/tmp/calculator.sock"})
server.ServeListener(l)
\n\n

subscriptions

The package also supports the publish subscribe pattern through the use of subscriptions.
A method that is considered eligible for notifications must satisfy the following
criteria:

\n
    \n
  • method must be exported
    • \n
  • first method argument type must be context.Context
    • \n
  • method must have return types (rpc.Subscription, error)
    • \n
\n

An example method:

\n
1
2
3
func (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {
\t\t...
\t}
\n\n

Reverse Calls

In any method handler, an instance of rpc.Client can be accessed through the ClientFromContext method. Using this client instance, server-to-client method calls can be performed on the RPC connection.

\n

server

to start rpc service, the invoking chain is as below

\n
1
node/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]
\n\n

API registration

"},{"title":"geth start","date":"2022-11-01T10:15:12.000Z","_content":"\n## build from source\n```\ngit clone https://github.com/ethereum/go-ethereum.git\ncd go-ethereum\nmake geth\n```\n\n## understanding geth config\ngeth config type is defined in /cmd/geth/config.go\n```go\ntype gethConfig struct {\n\tEth ethconfig.Config\n\tNode node.Config\n\tEthstats ethstatsConfig\n\tMetrics metrics.Config\n}\n```\n- **ethconfig** (eth/ethconfig/config.go)\ncontains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options\n- **nodeConfig** (node/config.go)\nrepresents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost\n- **metrics.Config** (metrics/config.go)\ncontains the configuration for the metric collection, such as InfluxDBEndpoint, etc\n- **ethstatsConfig**\nonly one URL entry\n\ngeth provides default config in the above files. user config file path is given by the below flag\n```go\nconfigFileFlag = &cli.StringFlag{\n\t\tName: \"config\",\n\t\tUsage: \"TOML configuration file\",\n\t\tCategory: flags.EthCategory,\n\t}\n```\n\nThe config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:\n```\n./geth --sepolia dumpconfig > geth-config.toml\n```\nto specify path to config file\n```\ngeth --sepolia --config geth-config.toml\n```\n\n## key configs\n- [Eth].TxLookupLimit \nNumber of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000\n- [Node].BootstrapNodes\nused to establish connectivity with the rest of the network.\ngeth provides default bootstrapNodes in file `params/bootnodes.go`\n- [Metrics_AND_STATS].ethstats\nReporting URL of a ethstats service (nodename:secret@host:port), [more detail](https://geth.ethereum.org/docs/monitoring/ethstats)\n- SyncMode\n- TrieDirtyCache\n- NoPruning\n- TrieCleanCacheJournal e.g triecache\n## how geth starts\n\n![geth starts](/images/geth_starts.drawio.png)\nthe main func is in `cmd/geth/main.go`\n```go\nfunc main() {\n\tif err := app.Run(os.Args); err != nil {\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}\n}\n```\nthe main() function is very short, and its main function is to start a tool for parsing command line commands: `gopkg.in/urfave/cli.v1`. Going deeper, we will find that `app.Action = geth` will be called when the cli app is initialized to call the geth() function\n```go\nfunc init() {\n\t// Initialize the CLI app and start Geth\n\tapp.Action = geth\n // ....\n}\n```\ngeth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.\n```go\nfunc geth(ctx *cli.Context) error {\n\tif args := ctx.Args().Slice(); len(args) > 0 {\n\t\treturn fmt.Errorf(\"invalid command: %q\", args[0])\n\t}\n\n\tprepare(ctx)\n\tstack, backend := makeFullNode(ctx)\n\tdefer stack.Close()\n\n\tstartNode(ctx, stack, backend, false)\n\tstack.Wait()\n\treturn nil\n}\n```\nIn the geth() function, there are three important function calls, namely: `prepare()`, `makeFullNode()`, and `startNode()`.\n\n### prepare\nThe implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.\n\n### makeFullNode\nThe implementation of the `makeFullNode()` function is located in the `cmd/geth/config.go` file. It will load the context of the command and apply user given configuration; and generate instances of `stack` and `backend`. Among them, `stack` is an instance of `Node` type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the `node/node.go` file), which is initialized by calling `makeConfigNode()` function through `makeFullNode()` function. inside `makeFullNode`, it calls `node.New(&cfg.Node)` to initiate a node. During instantiating of node, it invokes `rpc.NewServer()` to create a new rpc server and put in the field `inprocHandler`. it registers `rpc` api namespace by default.\n\nThe `backend` here is an interface of `ethapi.Backend` type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in `internal/ethapi/backend.go`. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. `backend` is created by calling `backend, eth := utils.RegisterEthService(stack, &cfg.Eth)`. Inside, it calls `eth.New(stack, cfg)` to create `backend` instance. During `backend` initiating, it opens database (`chainDb, err := stack.OpenDatabaseWithFreezer(\"chaindata\", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, \"eth/db/chaindata/\", false)`). Further, it creates consensus engine, `engine := ethconfig.CreateConsensusEngine(stack, ðashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb)`. goerli testnet use POA consensus (clique). \n```go\ntype Backend interface {\n\tSyncProgress() ethereum.SyncProgress\n\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)\n\tChainDb() ethdb.Database\n\tAccountManager() *accounts.Manager\n\tExtRPCEnabled() bool\n\tRPCGasCap() uint64 // global gas cap for eth_call over rpc: DoS protection\n\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection\n\tRPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs\n\tUnprotectedAllowed() bool // allows only for EIP155 transactions.\n\tSetHead(number uint64)\n\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)\n\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)\n\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)\n\tCurrentHeader() *types.Header\n\tCurrentBlock() *types.Header\n\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)\n\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)\n\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)\n\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)\n\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)\n\tPendingBlockAndReceipts() (*types.Block, types.Receipts)\n\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)\n\tGetTd(ctx context.Context, hash common.Hash) *big.Int\n\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)\n\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription\n\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription\n\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription\n\tSendTx(ctx context.Context, signedTx *types.Transaction) error\n\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)\n\tGetPoolTransactions() (types.Transactions, error)\n\tGetPoolTransaction(txHash common.Hash) *types.Transaction\n\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)\n\tStats() (pending int, queued int)\n\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)\n\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)\n\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription\n\tChainConfig() *params.ChainConfig\n\tEngine() consensus.Engine\n\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)\n\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)\n\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription\n\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tBloomStatus() (uint64, uint64)\n\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)\n}\n```\n\nIf readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend\n\n### startNode\nThe last key function, `startNode()`, is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in `Node.lifecycles` and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function\n\nAt the end of the geth() function, the function executes `stack.Wait()`, so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance\n\n## Node\nAs we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service\n```go\ntype Node struct {\n\teventmux *event.TypeMux\n\tconfig *Config\n\taccman *accounts.Manager\n\tlog log.Logger\n\tkeyDir string // key store directory\n\tkeyDirTemp bool // If true, key directory will be removed by Stop\n\tdirLock *flock.Flock // prevents concurrent use of instance directory\n\tstop chan struct{} // Channel to wait for termination notifications\n\tserver *p2p.Server // Currently running P2P networking layer\n\tstartStopLock sync.Mutex // Start/Stop are protected by an additional lock\n\tstate int // Tracks state of node lifecycle\n\n\tlock sync.Mutex\n\tlifecycles []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle\n\trpcAPIs []rpc.API // List of APIs currently provided by the node\n\thttp *httpServer //\n\tws *httpServer //\n\thttpAuth *httpServer //\n\twsAuth *httpServer //\n\tipc *ipcServer // Stores information about the ipc http server\n\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests\n\n\tdatabases map[*closeTrackingDB]struct{} // All open databases\n}\n```\n\n### close node\nAs mentioned earlier, the main thread of the entire program is blocked because of calling `stack.Wait()`. We can see that a channel called `stop` is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently\n```go\n// Wait blocks until the node is closed.\nfunc (n *Node) Wait() {\n <-n.stop\n}\n```\nWhen the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.\nIt is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.\n```go\n// doClose releases resources acquired by New(), collecting errors.\nfunc (n *Node) doClose(errs []error) error {\n // Close databases. This needs the lock because it needs to\n // synchronize with OpenDatabase*.\n n.lock.Lock()\n n.state = closedState\n errs = append(errs, n.closeDatabases()...)\n n.lock.Unlock()\n\n if err := n.accman.Close(); err != nil {\n errs = append(errs, err)\n }\n if n.keyDirTemp {\n if err := os.RemoveAll(n.keyDir); err != nil {\n errs = append(errs, err)\n }\n }\n\n // Release instance directory lock.\n n.closeDataDir()\n\n // Unblock n.Wait.\n close(n.stop)\n\n // Report any errors that might have occurred.\n switch len(errs) {\n case 0:\n return nil\n case 1:\n return errs[0]\n default:\n return fmt.Errorf(\"%v\", errs)\n }\n}\n```\n\n## Ethereum Backend\nWe can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.\n```go\ntype Ethereum struct {\n\tconfig *ethconfig.Config\n\n\t// Handlers\n\ttxPool *txpool.TxPool\n\tblockchain *core.BlockChain\n\thandler *handler\n\tethDialCandidates enode.Iterator\n\tsnapDialCandidates enode.Iterator\n\tmerger *consensus.Merger\n\n\t// DB interfaces\n\tchainDb ethdb.Database // Block chain database\n\n\teventMux *event.TypeMux\n\tengine consensus.Engine\n\taccountManager *accounts.Manager\n\n\tbloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests\n\tbloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports\n\tcloseBloomHandler chan struct{}\n\n\tAPIBackend *EthAPIBackend\n\n\tminer *miner.Miner\n\tgasPrice *big.Int\n\tetherbase common.Address\n\n\tnetworkID uint64\n\tnetRPCService *ethapi.NetAPI\n\n\tp2pServer *p2p.Server\n\n\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)\n\n\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully\n}\n```\nNodes start and stop Mining by calling `Ethereum.StartMining()` and `Ethereum.StopMining()`. Setting the profit account of Mining is achieved by calling `Ethereum.SetEtherbase()`\nHere we pay extra attention to the member variable `handler`. The definition of `handler` is in `eth/handler.go`.\nFrom a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, `downloader.Downloader` is responsible for synchronizing Block from the network, and `fetcher.TxFetcher` is responsible for synchronizing transactions from the network","source":"_posts/geth/code_analysis/geth.0.get.start.md","raw":"---\ntitle: geth start\ndate: 2022-11-01 18:15:12\ntags: [blockchain,geth]\n---\n\n## build from source\n```\ngit clone https://github.com/ethereum/go-ethereum.git\ncd go-ethereum\nmake geth\n```\n\n## understanding geth config\ngeth config type is defined in /cmd/geth/config.go\n```go\ntype gethConfig struct {\n\tEth ethconfig.Config\n\tNode node.Config\n\tEthstats ethstatsConfig\n\tMetrics metrics.Config\n}\n```\n- **ethconfig** (eth/ethconfig/config.go)\ncontains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options\n- **nodeConfig** (node/config.go)\nrepresents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost\n- **metrics.Config** (metrics/config.go)\ncontains the configuration for the metric collection, such as InfluxDBEndpoint, etc\n- **ethstatsConfig**\nonly one URL entry\n\ngeth provides default config in the above files. user config file path is given by the below flag\n```go\nconfigFileFlag = &cli.StringFlag{\n\t\tName: \"config\",\n\t\tUsage: \"TOML configuration file\",\n\t\tCategory: flags.EthCategory,\n\t}\n```\n\nThe config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:\n```\n./geth --sepolia dumpconfig > geth-config.toml\n```\nto specify path to config file\n```\ngeth --sepolia --config geth-config.toml\n```\n\n## key configs\n- [Eth].TxLookupLimit \nNumber of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000\n- [Node].BootstrapNodes\nused to establish connectivity with the rest of the network.\ngeth provides default bootstrapNodes in file `params/bootnodes.go`\n- [Metrics_AND_STATS].ethstats\nReporting URL of a ethstats service (nodename:secret@host:port), [more detail](https://geth.ethereum.org/docs/monitoring/ethstats)\n- SyncMode\n- TrieDirtyCache\n- NoPruning\n- TrieCleanCacheJournal e.g triecache\n## how geth starts\n\n![geth starts](/images/geth_starts.drawio.png)\nthe main func is in `cmd/geth/main.go`\n```go\nfunc main() {\n\tif err := app.Run(os.Args); err != nil {\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}\n}\n```\nthe main() function is very short, and its main function is to start a tool for parsing command line commands: `gopkg.in/urfave/cli.v1`. Going deeper, we will find that `app.Action = geth` will be called when the cli app is initialized to call the geth() function\n```go\nfunc init() {\n\t// Initialize the CLI app and start Geth\n\tapp.Action = geth\n // ....\n}\n```\ngeth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.\n```go\nfunc geth(ctx *cli.Context) error {\n\tif args := ctx.Args().Slice(); len(args) > 0 {\n\t\treturn fmt.Errorf(\"invalid command: %q\", args[0])\n\t}\n\n\tprepare(ctx)\n\tstack, backend := makeFullNode(ctx)\n\tdefer stack.Close()\n\n\tstartNode(ctx, stack, backend, false)\n\tstack.Wait()\n\treturn nil\n}\n```\nIn the geth() function, there are three important function calls, namely: `prepare()`, `makeFullNode()`, and `startNode()`.\n\n### prepare\nThe implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.\n\n### makeFullNode\nThe implementation of the `makeFullNode()` function is located in the `cmd/geth/config.go` file. It will load the context of the command and apply user given configuration; and generate instances of `stack` and `backend`. Among them, `stack` is an instance of `Node` type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the `node/node.go` file), which is initialized by calling `makeConfigNode()` function through `makeFullNode()` function. inside `makeFullNode`, it calls `node.New(&cfg.Node)` to initiate a node. During instantiating of node, it invokes `rpc.NewServer()` to create a new rpc server and put in the field `inprocHandler`. it registers `rpc` api namespace by default.\n\nThe `backend` here is an interface of `ethapi.Backend` type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in `internal/ethapi/backend.go`. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. `backend` is created by calling `backend, eth := utils.RegisterEthService(stack, &cfg.Eth)`. Inside, it calls `eth.New(stack, cfg)` to create `backend` instance. During `backend` initiating, it opens database (`chainDb, err := stack.OpenDatabaseWithFreezer(\"chaindata\", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, \"eth/db/chaindata/\", false)`). Further, it creates consensus engine, `engine := ethconfig.CreateConsensusEngine(stack, ðashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb)`. goerli testnet use POA consensus (clique). \n```go\ntype Backend interface {\n\tSyncProgress() ethereum.SyncProgress\n\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)\n\tChainDb() ethdb.Database\n\tAccountManager() *accounts.Manager\n\tExtRPCEnabled() bool\n\tRPCGasCap() uint64 // global gas cap for eth_call over rpc: DoS protection\n\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection\n\tRPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs\n\tUnprotectedAllowed() bool // allows only for EIP155 transactions.\n\tSetHead(number uint64)\n\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)\n\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)\n\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)\n\tCurrentHeader() *types.Header\n\tCurrentBlock() *types.Header\n\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)\n\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)\n\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)\n\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)\n\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)\n\tPendingBlockAndReceipts() (*types.Block, types.Receipts)\n\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)\n\tGetTd(ctx context.Context, hash common.Hash) *big.Int\n\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)\n\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription\n\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription\n\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription\n\tSendTx(ctx context.Context, signedTx *types.Transaction) error\n\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)\n\tGetPoolTransactions() (types.Transactions, error)\n\tGetPoolTransaction(txHash common.Hash) *types.Transaction\n\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)\n\tStats() (pending int, queued int)\n\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)\n\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)\n\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription\n\tChainConfig() *params.ChainConfig\n\tEngine() consensus.Engine\n\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)\n\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)\n\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription\n\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tBloomStatus() (uint64, uint64)\n\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)\n}\n```\n\nIf readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend\n\n### startNode\nThe last key function, `startNode()`, is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in `Node.lifecycles` and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function\n\nAt the end of the geth() function, the function executes `stack.Wait()`, so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance\n\n## Node\nAs we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service\n```go\ntype Node struct {\n\teventmux *event.TypeMux\n\tconfig *Config\n\taccman *accounts.Manager\n\tlog log.Logger\n\tkeyDir string // key store directory\n\tkeyDirTemp bool // If true, key directory will be removed by Stop\n\tdirLock *flock.Flock // prevents concurrent use of instance directory\n\tstop chan struct{} // Channel to wait for termination notifications\n\tserver *p2p.Server // Currently running P2P networking layer\n\tstartStopLock sync.Mutex // Start/Stop are protected by an additional lock\n\tstate int // Tracks state of node lifecycle\n\n\tlock sync.Mutex\n\tlifecycles []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle\n\trpcAPIs []rpc.API // List of APIs currently provided by the node\n\thttp *httpServer //\n\tws *httpServer //\n\thttpAuth *httpServer //\n\twsAuth *httpServer //\n\tipc *ipcServer // Stores information about the ipc http server\n\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests\n\n\tdatabases map[*closeTrackingDB]struct{} // All open databases\n}\n```\n\n### close node\nAs mentioned earlier, the main thread of the entire program is blocked because of calling `stack.Wait()`. We can see that a channel called `stop` is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently\n```go\n// Wait blocks until the node is closed.\nfunc (n *Node) Wait() {\n <-n.stop\n}\n```\nWhen the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.\nIt is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.\n```go\n// doClose releases resources acquired by New(), collecting errors.\nfunc (n *Node) doClose(errs []error) error {\n // Close databases. This needs the lock because it needs to\n // synchronize with OpenDatabase*.\n n.lock.Lock()\n n.state = closedState\n errs = append(errs, n.closeDatabases()...)\n n.lock.Unlock()\n\n if err := n.accman.Close(); err != nil {\n errs = append(errs, err)\n }\n if n.keyDirTemp {\n if err := os.RemoveAll(n.keyDir); err != nil {\n errs = append(errs, err)\n }\n }\n\n // Release instance directory lock.\n n.closeDataDir()\n\n // Unblock n.Wait.\n close(n.stop)\n\n // Report any errors that might have occurred.\n switch len(errs) {\n case 0:\n return nil\n case 1:\n return errs[0]\n default:\n return fmt.Errorf(\"%v\", errs)\n }\n}\n```\n\n## Ethereum Backend\nWe can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.\n```go\ntype Ethereum struct {\n\tconfig *ethconfig.Config\n\n\t// Handlers\n\ttxPool *txpool.TxPool\n\tblockchain *core.BlockChain\n\thandler *handler\n\tethDialCandidates enode.Iterator\n\tsnapDialCandidates enode.Iterator\n\tmerger *consensus.Merger\n\n\t// DB interfaces\n\tchainDb ethdb.Database // Block chain database\n\n\teventMux *event.TypeMux\n\tengine consensus.Engine\n\taccountManager *accounts.Manager\n\n\tbloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests\n\tbloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports\n\tcloseBloomHandler chan struct{}\n\n\tAPIBackend *EthAPIBackend\n\n\tminer *miner.Miner\n\tgasPrice *big.Int\n\tetherbase common.Address\n\n\tnetworkID uint64\n\tnetRPCService *ethapi.NetAPI\n\n\tp2pServer *p2p.Server\n\n\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)\n\n\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully\n}\n```\nNodes start and stop Mining by calling `Ethereum.StartMining()` and `Ethereum.StopMining()`. Setting the profit account of Mining is achieved by calling `Ethereum.SetEtherbase()`\nHere we pay extra attention to the member variable `handler`. The definition of `handler` is in `eth/handler.go`.\nFrom a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, `downloader.Downloader` is responsible for synchronizing Block from the network, and `fetcher.TxFetcher` is responsible for synchronizing transactions from the network","slug":"geth/code_analysis/geth.0.get.start","published":1,"updated":"2023-04-25T14:59:44.499Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzt001nofsj44p54fok","content":"

build from source

1
2
3
git clone https://github.com/ethereum/go-ethereum.git
cd go-ethereum
make geth
\n\n

understanding geth config

geth config type is defined in /cmd/geth/config.go

\n
1
2
3
4
5
6
type gethConfig struct {
\tEth      ethconfig.Config
\tNode     node.Config
\tEthstats ethstatsConfig
\tMetrics  metrics.Config
}
\n
    \n
  • ethconfig (eth/ethconfig/config.go)
    contains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options
    • \n
  • nodeConfig (node/config.go)
    represents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost
    • \n
  • metrics.Config (metrics/config.go)
    contains the configuration for the metric collection, such as InfluxDBEndpoint, etc
    • \n
  • ethstatsConfig
    only one URL entry
    • \n
\n

geth provides default config in the above files. user config file path is given by the below flag

\n
1
2
3
4
5
configFileFlag = &cli.StringFlag{
\t\tName:     "config",
\t\tUsage:    "TOML configuration file",
\t\tCategory: flags.EthCategory,
\t}
\n\n

The config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:

\n
1
./geth --sepolia dumpconfig > geth-config.toml
\n

to specify path to config file

\n
1
geth --sepolia --config geth-config.toml
\n\n

key configs

    \n
  • [Eth].TxLookupLimit
    Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000
    • \n
  • [Node].BootstrapNodes
    used to establish connectivity with the rest of the network.
    geth provides default bootstrapNodes in file params/bootnodes.go
    • \n
  • [Metrics_AND_STATS].ethstats
    Reporting URL of a ethstats service (nodename:secret@host:port), more detail
    • \n
  • SyncMode
    • \n
  • TrieDirtyCache
    • \n
  • NoPruning
    • \n
  • TrieCleanCacheJournal e.g triecache
    • \n
\n

how geth starts

\"geth
the main func is in cmd/geth/main.go

\n
1
2
3
4
5
6
func main() {
\tif err := app.Run(os.Args); err != nil {
\t\tfmt.Fprintln(os.Stderr, err)
\t\tos.Exit(1)
\t}
}
\n

the main() function is very short, and its main function is to start a tool for parsing command line commands: gopkg.in/urfave/cli.v1. Going deeper, we will find that app.Action = geth will be called when the cli app is initialized to call the geth() function

\n
1
2
3
4
5
func init() {
\t// Initialize the CLI app and start Geth
\tapp.Action = geth
    // ....
}
\n

geth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
func geth(ctx *cli.Context) error {
\tif args := ctx.Args().Slice(); len(args) > 0 {
\t\treturn fmt.Errorf("invalid command: %q", args[0])
\t}

\tprepare(ctx)
\tstack, backend := makeFullNode(ctx)
\tdefer stack.Close()

\tstartNode(ctx, stack, backend, false)
\tstack.Wait()
\treturn nil
}
\n

In the geth() function, there are three important function calls, namely: prepare(), makeFullNode(), and startNode().

\n

prepare

The implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.

\n

makeFullNode

The implementation of the makeFullNode() function is located in the cmd/geth/config.go file. It will load the context of the command and apply user given configuration; and generate instances of stack and backend. Among them, stack is an instance of Node type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the node/node.go file), which is initialized by calling makeConfigNode() function through makeFullNode() function. inside makeFullNode, it calls node.New(&cfg.Node) to initiate a node. During instantiating of node, it invokes rpc.NewServer() to create a new rpc server and put in the field inprocHandler. it registers rpc api namespace by default.

\n

The backend here is an interface of ethapi.Backend type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in internal/ethapi/backend.go. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. backend is created by calling backend, eth := utils.RegisterEthService(stack, &cfg.Eth). Inside, it calls eth.New(stack, cfg) to create backend instance. During backend initiating, it opens database (chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)). Further, it creates consensus engine, engine := ethconfig.CreateConsensusEngine(stack, &ethashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb). goerli testnet use POA consensus (clique).

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
type Backend interface {
\tSyncProgress() ethereum.SyncProgress
\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)
\tChainDb() ethdb.Database
\tAccountManager() *accounts.Manager
\tExtRPCEnabled() bool
\tRPCGasCap() uint64            // global gas cap for eth_call over rpc: DoS protection
\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection
\tRPCTxFeeCap() float64         // global tx fee cap for all transaction related APIs
\tUnprotectedAllowed() bool     // allows only for EIP155 transactions.
\tSetHead(number uint64)
\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)
\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)
\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)
\tCurrentHeader() *types.Header
\tCurrentBlock() *types.Header
\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)
\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)
\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)
\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)
\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)
\tPendingBlockAndReceipts() (*types.Block, types.Receipts)
\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)
\tGetTd(ctx context.Context, hash common.Hash) *big.Int
\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)
\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription
\tSendTx(ctx context.Context, signedTx *types.Transaction) error
\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)
\tGetPoolTransactions() (types.Transactions, error)
\tGetPoolTransaction(txHash common.Hash) *types.Transaction
\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)
\tStats() (pending int, queued int)
\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)
\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)
\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
\tChainConfig() *params.ChainConfig
\tEngine() consensus.Engine
\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)
\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)
\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
\tBloomStatus() (uint64, uint64)
\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
}
\n\n

If readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend

\n

startNode

The last key function, startNode(), is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in Node.lifecycles and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function

\n

At the end of the geth() function, the function executes stack.Wait(), so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance

\n

Node

As we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
type Node struct {
\teventmux      *event.TypeMux
\tconfig        *Config
\taccman        *accounts.Manager
\tlog           log.Logger
\tkeyDir        string        // key store directory
\tkeyDirTemp    bool          // If true, key directory will be removed by Stop
\tdirLock       *flock.Flock  // prevents concurrent use of instance directory
\tstop          chan struct{} // Channel to wait for termination notifications
\tserver        *p2p.Server   // Currently running P2P networking layer
\tstartStopLock sync.Mutex    // Start/Stop are protected by an additional lock
\tstate         int           // Tracks state of node lifecycle

\tlock          sync.Mutex
\tlifecycles    []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle
\trpcAPIs       []rpc.API   // List of APIs currently provided by the node
\thttp          *httpServer //
\tws            *httpServer //
\thttpAuth      *httpServer //
\twsAuth        *httpServer //
\tipc           *ipcServer  // Stores information about the ipc http server
\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests

\tdatabases map[*closeTrackingDB]struct{} // All open databases
}
\n\n

close node

As mentioned earlier, the main thread of the entire program is blocked because of calling stack.Wait(). We can see that a channel called stop is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently

\n
1
2
3
4
// Wait blocks until the node is closed.
func (n *Node) Wait() {
 <-n.stop
}
\n

When the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.
It is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
// doClose releases resources acquired by New(), collecting errors.
func (n *Node) doClose(errs []error) error {
 // Close databases. This needs the lock because it needs to
 // synchronize with OpenDatabase*.
 n.lock.Lock()
 n.state = closedState
 errs = append(errs, n.closeDatabases()...)
 n.lock.Unlock()

 if err := n.accman.Close(); err != nil {
  errs = append(errs, err)
 }
 if n.keyDirTemp {
  if err := os.RemoveAll(n.keyDir); err != nil {
   errs = append(errs, err)
  }
 }

 // Release instance directory lock.
 n.closeDataDir()

 // Unblock n.Wait.
 close(n.stop)

 // Report any errors that might have occurred.
 switch len(errs) {
 case 0:
  return nil
 case 1:
  return errs[0]
 default:
  return fmt.Errorf("%v", errs)
 }
}
\n\n

Ethereum Backend

We can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
type Ethereum struct {
\tconfig *ethconfig.Config

\t// Handlers
\ttxPool             *txpool.TxPool
\tblockchain         *core.BlockChain
\thandler            *handler
\tethDialCandidates  enode.Iterator
\tsnapDialCandidates enode.Iterator
\tmerger             *consensus.Merger

\t// DB interfaces
\tchainDb ethdb.Database // Block chain database

\teventMux       *event.TypeMux
\tengine         consensus.Engine
\taccountManager *accounts.Manager

\tbloomRequests     chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests
\tbloomIndexer      *core.ChainIndexer             // Bloom indexer operating during block imports
\tcloseBloomHandler chan struct{}

\tAPIBackend *EthAPIBackend

\tminer     *miner.Miner
\tgasPrice  *big.Int
\tetherbase common.Address

\tnetworkID     uint64
\tnetRPCService *ethapi.NetAPI

\tp2pServer *p2p.Server

\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)

\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
}
\n

Nodes start and stop Mining by calling Ethereum.StartMining() and Ethereum.StopMining(). Setting the profit account of Mining is achieved by calling Ethereum.SetEtherbase()
Here we pay extra attention to the member variable handler. The definition of handler is in eth/handler.go.
From a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, downloader.Downloader is responsible for synchronizing Block from the network, and fetcher.TxFetcher is responsible for synchronizing transactions from the network

\n","site":{"data":{}},"excerpt":"","more":"

build from source

1
2
3
git clone https://github.com/ethereum/go-ethereum.git
cd go-ethereum
make geth
\n\n

understanding geth config

geth config type is defined in /cmd/geth/config.go

\n
1
2
3
4
5
6
type gethConfig struct {
\tEth      ethconfig.Config
\tNode     node.Config
\tEthstats ethstatsConfig
\tMetrics  metrics.Config
}
\n
    \n
  • ethconfig (eth/ethconfig/config.go)
    contains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options
    • \n
  • nodeConfig (node/config.go)
    represents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost
    • \n
  • metrics.Config (metrics/config.go)
    contains the configuration for the metric collection, such as InfluxDBEndpoint, etc
    • \n
  • ethstatsConfig
    only one URL entry
    • \n
\n

geth provides default config in the above files. user config file path is given by the below flag

\n
1
2
3
4
5
configFileFlag = &cli.StringFlag{
\t\tName:     "config",
\t\tUsage:    "TOML configuration file",
\t\tCategory: flags.EthCategory,
\t}
\n\n

The config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:

\n
1
./geth --sepolia dumpconfig > geth-config.toml
\n

to specify path to config file

\n
1
geth --sepolia --config geth-config.toml
\n\n

key configs

    \n
  • [Eth].TxLookupLimit
    Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000
    • \n
  • [Node].BootstrapNodes
    used to establish connectivity with the rest of the network.
    geth provides default bootstrapNodes in file params/bootnodes.go
    • \n
  • [Metrics_AND_STATS].ethstats
    Reporting URL of a ethstats service (nodename:secret@host:port), more detail
    • \n
  • SyncMode
    • \n
  • TrieDirtyCache
    • \n
  • NoPruning
    • \n
  • TrieCleanCacheJournal e.g triecache
    • \n
\n

how geth starts

\"geth
the main func is in cmd/geth/main.go

\n
1
2
3
4
5
6
func main() {
\tif err := app.Run(os.Args); err != nil {
\t\tfmt.Fprintln(os.Stderr, err)
\t\tos.Exit(1)
\t}
}
\n

the main() function is very short, and its main function is to start a tool for parsing command line commands: gopkg.in/urfave/cli.v1. Going deeper, we will find that app.Action = geth will be called when the cli app is initialized to call the geth() function

\n
1
2
3
4
5
func init() {
\t// Initialize the CLI app and start Geth
\tapp.Action = geth
    // ....
}
\n

geth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
func geth(ctx *cli.Context) error {
\tif args := ctx.Args().Slice(); len(args) > 0 {
\t\treturn fmt.Errorf("invalid command: %q", args[0])
\t}

\tprepare(ctx)
\tstack, backend := makeFullNode(ctx)
\tdefer stack.Close()

\tstartNode(ctx, stack, backend, false)
\tstack.Wait()
\treturn nil
}
\n

In the geth() function, there are three important function calls, namely: prepare(), makeFullNode(), and startNode().

\n

prepare

The implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.

\n

makeFullNode

The implementation of the makeFullNode() function is located in the cmd/geth/config.go file. It will load the context of the command and apply user given configuration; and generate instances of stack and backend. Among them, stack is an instance of Node type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the node/node.go file), which is initialized by calling makeConfigNode() function through makeFullNode() function. inside makeFullNode, it calls node.New(&cfg.Node) to initiate a node. During instantiating of node, it invokes rpc.NewServer() to create a new rpc server and put in the field inprocHandler. it registers rpc api namespace by default.

\n

The backend here is an interface of ethapi.Backend type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in internal/ethapi/backend.go. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. backend is created by calling backend, eth := utils.RegisterEthService(stack, &cfg.Eth). Inside, it calls eth.New(stack, cfg) to create backend instance. During backend initiating, it opens database (chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)). Further, it creates consensus engine, engine := ethconfig.CreateConsensusEngine(stack, &ethashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb). goerli testnet use POA consensus (clique).

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
type Backend interface {
\tSyncProgress() ethereum.SyncProgress
\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)
\tChainDb() ethdb.Database
\tAccountManager() *accounts.Manager
\tExtRPCEnabled() bool
\tRPCGasCap() uint64            // global gas cap for eth_call over rpc: DoS protection
\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection
\tRPCTxFeeCap() float64         // global tx fee cap for all transaction related APIs
\tUnprotectedAllowed() bool     // allows only for EIP155 transactions.
\tSetHead(number uint64)
\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)
\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)
\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)
\tCurrentHeader() *types.Header
\tCurrentBlock() *types.Header
\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)
\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)
\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)
\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)
\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)
\tPendingBlockAndReceipts() (*types.Block, types.Receipts)
\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)
\tGetTd(ctx context.Context, hash common.Hash) *big.Int
\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)
\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription
\tSendTx(ctx context.Context, signedTx *types.Transaction) error
\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)
\tGetPoolTransactions() (types.Transactions, error)
\tGetPoolTransaction(txHash common.Hash) *types.Transaction
\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)
\tStats() (pending int, queued int)
\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)
\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)
\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
\tChainConfig() *params.ChainConfig
\tEngine() consensus.Engine
\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)
\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)
\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
\tBloomStatus() (uint64, uint64)
\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
}
\n\n

If readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend

\n

startNode

The last key function, startNode(), is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in Node.lifecycles and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function

\n

At the end of the geth() function, the function executes stack.Wait(), so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance

\n

Node

As we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
type Node struct {
\teventmux      *event.TypeMux
\tconfig        *Config
\taccman        *accounts.Manager
\tlog           log.Logger
\tkeyDir        string        // key store directory
\tkeyDirTemp    bool          // If true, key directory will be removed by Stop
\tdirLock       *flock.Flock  // prevents concurrent use of instance directory
\tstop          chan struct{} // Channel to wait for termination notifications
\tserver        *p2p.Server   // Currently running P2P networking layer
\tstartStopLock sync.Mutex    // Start/Stop are protected by an additional lock
\tstate         int           // Tracks state of node lifecycle

\tlock          sync.Mutex
\tlifecycles    []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle
\trpcAPIs       []rpc.API   // List of APIs currently provided by the node
\thttp          *httpServer //
\tws            *httpServer //
\thttpAuth      *httpServer //
\twsAuth        *httpServer //
\tipc           *ipcServer  // Stores information about the ipc http server
\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests

\tdatabases map[*closeTrackingDB]struct{} // All open databases
}
\n\n

close node

As mentioned earlier, the main thread of the entire program is blocked because of calling stack.Wait(). We can see that a channel called stop is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently

\n
1
2
3
4
// Wait blocks until the node is closed.
func (n *Node) Wait() {
 <-n.stop
}
\n

When the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.
It is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
// doClose releases resources acquired by New(), collecting errors.
func (n *Node) doClose(errs []error) error {
 // Close databases. This needs the lock because it needs to
 // synchronize with OpenDatabase*.
 n.lock.Lock()
 n.state = closedState
 errs = append(errs, n.closeDatabases()...)
 n.lock.Unlock()

 if err := n.accman.Close(); err != nil {
  errs = append(errs, err)
 }
 if n.keyDirTemp {
  if err := os.RemoveAll(n.keyDir); err != nil {
   errs = append(errs, err)
  }
 }

 // Release instance directory lock.
 n.closeDataDir()

 // Unblock n.Wait.
 close(n.stop)

 // Report any errors that might have occurred.
 switch len(errs) {
 case 0:
  return nil
 case 1:
  return errs[0]
 default:
  return fmt.Errorf("%v", errs)
 }
}
\n\n

Ethereum Backend

We can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
type Ethereum struct {
\tconfig *ethconfig.Config

\t// Handlers
\ttxPool             *txpool.TxPool
\tblockchain         *core.BlockChain
\thandler            *handler
\tethDialCandidates  enode.Iterator
\tsnapDialCandidates enode.Iterator
\tmerger             *consensus.Merger

\t// DB interfaces
\tchainDb ethdb.Database // Block chain database

\teventMux       *event.TypeMux
\tengine         consensus.Engine
\taccountManager *accounts.Manager

\tbloomRequests     chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests
\tbloomIndexer      *core.ChainIndexer             // Bloom indexer operating during block imports
\tcloseBloomHandler chan struct{}

\tAPIBackend *EthAPIBackend

\tminer     *miner.Miner
\tgasPrice  *big.Int
\tetherbase common.Address

\tnetworkID     uint64
\tnetRPCService *ethapi.NetAPI

\tp2pServer *p2p.Server

\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)

\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
}
\n

Nodes start and stop Mining by calling Ethereum.StartMining() and Ethereum.StopMining(). Setting the profit account of Mining is achieved by calling Ethereum.SetEtherbase()
Here we pay extra attention to the member variable handler. The definition of handler is in eth/handler.go.
From a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, downloader.Downloader is responsible for synchronizing Block from the network, and fetcher.TxFetcher is responsible for synchronizing transactions from the network

\n"},{"title":"geth evm source analysis","date":"2023-01-08T08:24:54.000Z","_content":"\n# overall\nthe code is under path `core/vm`\noverview of the whole evm module ![evm](/images/evm.drawio.google.png)\n\nthe core is `EVM` struct (in evm.go), with main function in creating or call contract. a new `EVM` object is created every time when processing a transaction. inside the EVM struct, the main items are `Interpreter`, and `StateDB` (for state persistence). `Interpreter` loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in `JumpTable` (256 sized array of `operation`)\n\ndepending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.\n\n# evm\nThe `EVM` object is the most important object exported by the evm module, which represents an Ethereum virtual machine\n\n## creating evm\nEvery time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function `ApplyTransaction` (core/state_processor.go)\n\n## creating contract\nIf the `to` of the transaction is empty, it means that this transaction is to create a contract, so call `EVM.Create` to perform related functions\n- CREATE\n```\ncontractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))\n```\n- CREATE2\n```\ncodeAndHash := &codeAndHash{code: code}\n\tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())\n```\nduring create contract, an object `Contract` is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the `jumpdests` record of the code.\n\nthen, it invokes below method to create contract\n```\nret, err := evm.interpreter.Run(contract, nil, false)\nevm.StateDB.SetCode(address, ret)\n```\nIf the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.\n\nYou may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract's \"constructor\" (that is, the contract object's constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a `ret` variable here Stored in the state database as the actual code of the contract\n\n## call contract\nThe EVM object has three methods to implement the call of the contract, they are:\n\n- EVM. Call\n- EVM. CallCode\n- EVM. DelegateCall\n- EVM.StaticCall\nThe basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods\n\n### EVM.CallCode & EVM.DelegateCall\nThe existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the \"library\" of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this \"library contract\". But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the \"library contract\", these properties must be the properties of the \"library contract\" itself, but this may not be what we want\n\nas an example\n```\nA -> contractB - delegateCall -> libC\n```\n`EVM.DelegateCall` sets the caller (msg.sender) of the \"library contract\" (libC) to A, rather than contractB; sets the address of the \"library contract\" (libC) to contractB. \n`EVM.CallCode` is similar to `EVM.DelegateCall`. the only difference is that `EVM.CallCode` only change the address of the \"library contract\" (libC) to contractB, without chanding the caller to A.\n`EVM.StaticCall` is similar to `EVM.Call`, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data\n\nduring contract call, it first check whether it is precompiled contract. some precompiled contracts are\n- common.BytesToAddress([]byte{1}): &ecrecover{},\n- common.BytesToAddress([]byte{2}): &sha256hash{},\n- common.BytesToAddress([]byte{3}): &ripemd160hash{},\n- common.BytesToAddress([]byte{4}): &dataCopy{},\n\n# EVMInterpreter\nThe interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.\n\n## execution layout\n![layout](/images/evm.layout.png)\n\n## intrinsic gas\nThe intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.\n\n## gas cost\nthe gas cost of each instruction is stored in `JumpTable.operation.dynamicGas` or `JumpTable.operation.constantGas`. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.\n\nIn fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.\n\na method `memoryGasCost`is used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.\n\n# JumpTable\njumptable is 256 sized array of `operation`\n\n## jump instruction\nAmong the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST\n```\nfunc opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {\n pos := stack.pop()\n if !contract.validJumpdest(pos) {\n nop := contract.GetOp(pos.Uint64())\n return nil, fmt.Errorf(\"invalid jump destination (%v) %v\", nop, pos)\n }\n *pc = pos.Uint64()\n\n interpreter.intPool.put(pos)\n return nil, nil\n}\n```\nA function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.\n\nTo judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.\n\nLet's introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the \"bit\" corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the \"bit\" of the offset value of the jump destination in this bit vector object is 0\n\n# references\n- [yangzhe_blog](https://yangzhe.me/2019/08/12/ethereum-evm/#%E8%A7%A3%E9%87%8A%E5%99%A8%E5%AF%B9%E8%B1%A1evminterpreter)\n- [op code manual](https://www.evm.codes/?fork=shanghai)","source":"_posts/geth/code_analysis/geth.evm.md","raw":"---\ntitle: geth evm source analysis\ndate: 2023-01-08 16:24:54\ntags: [blockchain,geth]\n---\n\n# overall\nthe code is under path `core/vm`\noverview of the whole evm module ![evm](/images/evm.drawio.google.png)\n\nthe core is `EVM` struct (in evm.go), with main function in creating or call contract. a new `EVM` object is created every time when processing a transaction. inside the EVM struct, the main items are `Interpreter`, and `StateDB` (for state persistence). `Interpreter` loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in `JumpTable` (256 sized array of `operation`)\n\ndepending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.\n\n# evm\nThe `EVM` object is the most important object exported by the evm module, which represents an Ethereum virtual machine\n\n## creating evm\nEvery time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function `ApplyTransaction` (core/state_processor.go)\n\n## creating contract\nIf the `to` of the transaction is empty, it means that this transaction is to create a contract, so call `EVM.Create` to perform related functions\n- CREATE\n```\ncontractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))\n```\n- CREATE2\n```\ncodeAndHash := &codeAndHash{code: code}\n\tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())\n```\nduring create contract, an object `Contract` is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the `jumpdests` record of the code.\n\nthen, it invokes below method to create contract\n```\nret, err := evm.interpreter.Run(contract, nil, false)\nevm.StateDB.SetCode(address, ret)\n```\nIf the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.\n\nYou may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract's \"constructor\" (that is, the contract object's constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a `ret` variable here Stored in the state database as the actual code of the contract\n\n## call contract\nThe EVM object has three methods to implement the call of the contract, they are:\n\n- EVM. Call\n- EVM. CallCode\n- EVM. DelegateCall\n- EVM.StaticCall\nThe basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods\n\n### EVM.CallCode & EVM.DelegateCall\nThe existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the \"library\" of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this \"library contract\". But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the \"library contract\", these properties must be the properties of the \"library contract\" itself, but this may not be what we want\n\nas an example\n```\nA -> contractB - delegateCall -> libC\n```\n`EVM.DelegateCall` sets the caller (msg.sender) of the \"library contract\" (libC) to A, rather than contractB; sets the address of the \"library contract\" (libC) to contractB. \n`EVM.CallCode` is similar to `EVM.DelegateCall`. the only difference is that `EVM.CallCode` only change the address of the \"library contract\" (libC) to contractB, without chanding the caller to A.\n`EVM.StaticCall` is similar to `EVM.Call`, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data\n\nduring contract call, it first check whether it is precompiled contract. some precompiled contracts are\n- common.BytesToAddress([]byte{1}): &ecrecover{},\n- common.BytesToAddress([]byte{2}): &sha256hash{},\n- common.BytesToAddress([]byte{3}): &ripemd160hash{},\n- common.BytesToAddress([]byte{4}): &dataCopy{},\n\n# EVMInterpreter\nThe interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.\n\n## execution layout\n![layout](/images/evm.layout.png)\n\n## intrinsic gas\nThe intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.\n\n## gas cost\nthe gas cost of each instruction is stored in `JumpTable.operation.dynamicGas` or `JumpTable.operation.constantGas`. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.\n\nIn fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.\n\na method `memoryGasCost`is used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.\n\n# JumpTable\njumptable is 256 sized array of `operation`\n\n## jump instruction\nAmong the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST\n```\nfunc opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {\n pos := stack.pop()\n if !contract.validJumpdest(pos) {\n nop := contract.GetOp(pos.Uint64())\n return nil, fmt.Errorf(\"invalid jump destination (%v) %v\", nop, pos)\n }\n *pc = pos.Uint64()\n\n interpreter.intPool.put(pos)\n return nil, nil\n}\n```\nA function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.\n\nTo judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.\n\nLet's introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the \"bit\" corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the \"bit\" of the offset value of the jump destination in this bit vector object is 0\n\n# references\n- [yangzhe_blog](https://yangzhe.me/2019/08/12/ethereum-evm/#%E8%A7%A3%E9%87%8A%E5%99%A8%E5%AF%B9%E8%B1%A1evminterpreter)\n- [op code manual](https://www.evm.codes/?fork=shanghai)","slug":"geth/code_analysis/geth.evm","published":1,"updated":"2023-04-14T03:02:06.144Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzt001pofsj2eunaq89","content":"

overall

the code is under path core/vm
overview of the whole evm module \"evm\"

\n

the core is EVM struct (in evm.go), with main function in creating or call contract. a new EVM object is created every time when processing a transaction. inside the EVM struct, the main items are Interpreter, and StateDB (for state persistence). Interpreter loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in JumpTable (256 sized array of operation)

\n

depending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.

\n

evm

The EVM object is the most important object exported by the evm module, which represents an Ethereum virtual machine

\n

creating evm

Every time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function ApplyTransaction (core/state_processor.go)

\n

creating contract

If the to of the transaction is empty, it means that this transaction is to create a contract, so call EVM.Create to perform related functions

\n
    \n
  • CREATE
    1
    contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))
    • \n
  • CREATE2
    1
    2
    codeAndHash := &codeAndHash{code: code}
    \tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())
    \nduring create contract, an object Contract is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the jumpdests record of the code.
    • \n
\n

then, it invokes below method to create contract

\n
1
2
ret, err := evm.interpreter.Run(contract, nil, false)
evm.StateDB.SetCode(address, ret)
\n

If the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.

\n

You may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract’s “constructor” (that is, the contract object’s constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a ret variable here Stored in the state database as the actual code of the contract

\n

call contract

The EVM object has three methods to implement the call of the contract, they are:

\n
    \n
  • EVM. Call
    • \n
  • EVM. CallCode
    • \n
  • EVM. DelegateCall
    • \n
  • EVM.StaticCall
    The basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods
    • \n
\n

EVM.CallCode & EVM.DelegateCall

The existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the “library” of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this “library contract”. But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the “library contract”, these properties must be the properties of the “library contract” itself, but this may not be what we want

\n

as an example

\n
1
A -> contractB - delegateCall -> libC
\n

EVM.DelegateCall sets the caller (msg.sender) of the “library contract” (libC) to A, rather than contractB; sets the address of the “library contract” (libC) to contractB.
EVM.CallCode is similar to EVM.DelegateCall. the only difference is that EVM.CallCode only change the address of the “library contract” (libC) to contractB, without chanding the caller to A.
EVM.StaticCall is similar to EVM.Call, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data

\n

during contract call, it first check whether it is precompiled contract. some precompiled contracts are

\n
    \n
  • common.BytesToAddress([]byte{1}): &ecrecover{},
    • \n
  • common.BytesToAddress([]byte{2}): &sha256hash{},
    • \n
  • common.BytesToAddress([]byte{3}): &ripemd160hash{},
    • \n
  • common.BytesToAddress([]byte{4}): &dataCopy{},
    • \n
\n

EVMInterpreter

The interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.

\n

execution layout

\"layout\"

\n

intrinsic gas

The intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.

\n

gas cost

the gas cost of each instruction is stored in JumpTable.operation.dynamicGas or JumpTable.operation.constantGas. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.

\n

In fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.

\n

a method memoryGasCostis used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.

\n

JumpTable

jumptable is 256 sized array of operation

\n

jump instruction

Among the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST

\n
1
2
3
4
5
6
7
8
9
10
11
func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
    pos := stack.pop()
    if !contract.validJumpdest(pos) {
        nop := contract.GetOp(pos.Uint64())
        return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos)
    }
    *pc = pos.Uint64()

    interpreter.intPool.put(pos)
    return nil, nil
}
\n

A function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.

\n

To judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.

\n

Let’s introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the “bit” corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the “bit” of the offset value of the jump destination in this bit vector object is 0

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

overall

the code is under path core/vm
overview of the whole evm module \"evm\"

\n

the core is EVM struct (in evm.go), with main function in creating or call contract. a new EVM object is created every time when processing a transaction. inside the EVM struct, the main items are Interpreter, and StateDB (for state persistence). Interpreter loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in JumpTable (256 sized array of operation)

\n

depending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.

\n

evm

The EVM object is the most important object exported by the evm module, which represents an Ethereum virtual machine

\n

creating evm

Every time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function ApplyTransaction (core/state_processor.go)

\n

creating contract

If the to of the transaction is empty, it means that this transaction is to create a contract, so call EVM.Create to perform related functions

\n
    \n
  • CREATE
    1
    contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))
    • \n
  • CREATE2
    1
    2
    codeAndHash := &codeAndHash{code: code}
    \tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())
    \nduring create contract, an object Contract is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the jumpdests record of the code.
    • \n
\n

then, it invokes below method to create contract

\n
1
2
ret, err := evm.interpreter.Run(contract, nil, false)
evm.StateDB.SetCode(address, ret)
\n

If the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.

\n

You may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract’s “constructor” (that is, the contract object’s constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a ret variable here Stored in the state database as the actual code of the contract

\n

call contract

The EVM object has three methods to implement the call of the contract, they are:

\n
    \n
  • EVM. Call
    • \n
  • EVM. CallCode
    • \n
  • EVM. DelegateCall
    • \n
  • EVM.StaticCall
    The basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods
    • \n
\n

EVM.CallCode & EVM.DelegateCall

The existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the “library” of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this “library contract”. But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the “library contract”, these properties must be the properties of the “library contract” itself, but this may not be what we want

\n

as an example

\n
1
A -> contractB - delegateCall -> libC
\n

EVM.DelegateCall sets the caller (msg.sender) of the “library contract” (libC) to A, rather than contractB; sets the address of the “library contract” (libC) to contractB.
EVM.CallCode is similar to EVM.DelegateCall. the only difference is that EVM.CallCode only change the address of the “library contract” (libC) to contractB, without chanding the caller to A.
EVM.StaticCall is similar to EVM.Call, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data

\n

during contract call, it first check whether it is precompiled contract. some precompiled contracts are

\n
    \n
  • common.BytesToAddress([]byte{1}): &ecrecover{},
    • \n
  • common.BytesToAddress([]byte{2}): &sha256hash{},
    • \n
  • common.BytesToAddress([]byte{3}): &ripemd160hash{},
    • \n
  • common.BytesToAddress([]byte{4}): &dataCopy{},
    • \n
\n

EVMInterpreter

The interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.

\n

execution layout

\"layout\"

\n

intrinsic gas

The intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.

\n

gas cost

the gas cost of each instruction is stored in JumpTable.operation.dynamicGas or JumpTable.operation.constantGas. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.

\n

In fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.

\n

a method memoryGasCostis used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.

\n

JumpTable

jumptable is 256 sized array of operation

\n

jump instruction

Among the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST

\n
1
2
3
4
5
6
7
8
9
10
11
func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
    pos := stack.pop()
    if !contract.validJumpdest(pos) {
        nop := contract.GetOp(pos.Uint64())
        return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos)
    }
    *pc = pos.Uint64()

    interpreter.intPool.put(pos)
    return nil, nil
}
\n

A function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.

\n

To judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.

\n

Let’s introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the “bit” corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the “bit” of the offset value of the jump destination in this bit vector object is 0

\n

references

\n"},{"title":"rust frequently used crates","date":"2022-12-13T09:15:23.000Z","_content":"\n\ntokio-trace -> tracing\n\ncontexts, multi threads\ncausality -\nstructured diagnostics ( no grep)\n\ntracing is part of tokio, tokio not requied\n\nspans: a perido of time, entered and exited\nevents: singular moment in time\nsubscriber: collect trace","source":"_posts/rust/crates/rust-frequently-used-crates.md","raw":"---\ntitle: rust frequently used crates\ndate: 2022-12-13 17:15:23\ntags: [rust]\n---\n\n\ntokio-trace -> tracing\n\ncontexts, multi threads\ncausality -\nstructured diagnostics ( no grep)\n\ntracing is part of tokio, tokio not requied\n\nspans: a perido of time, entered and exited\nevents: singular moment in time\nsubscriber: collect trace","slug":"rust/crates/rust-frequently-used-crates","published":1,"updated":"2023-05-03T09:29:19.269Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzu001sofsjc0w8dddm","content":"

tokio-trace -> tracing

\n

contexts, multi threads
causality -
structured diagnostics ( no grep)

\n

tracing is part of tokio, tokio not requied

\n

spans: a perido of time, entered and exited
events: singular moment in time
subscriber: collect trace

\n","site":{"data":{}},"excerpt":"","more":"

tokio-trace -> tracing

\n

contexts, multi threads
causality -
structured diagnostics ( no grep)

\n

tracing is part of tokio, tokio not requied

\n

spans: a perido of time, entered and exited
events: singular moment in time
subscriber: collect trace

\n"},{"title":"rust crate serde","date":"2023-04-01T14:04:38.000Z","_content":"\n```rust\n#[serde(tag = \"filterType\")]\n#[serde(untagged)]\n#[serde(rename = \"PRICE_FILTER\")]\n#[serde(rename_all = \"camelCase\")]\n\n#[serde(with = \"string_or_float\")]\npub stop_price: f64,\n```","source":"_posts/rust/crates/rust-serde.md","raw":"---\ntitle: rust crate serde\ndate: 2023-04-01 22:04:38\ntags: [rust-crate]\n---\n\n```rust\n#[serde(tag = \"filterType\")]\n#[serde(untagged)]\n#[serde(rename = \"PRICE_FILTER\")]\n#[serde(rename_all = \"camelCase\")]\n\n#[serde(with = \"string_or_float\")]\npub stop_price: f64,\n```","slug":"rust/crates/rust-serde","published":1,"updated":"2023-06-29T15:48:46.930Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzu001uofsjhhpibdh9","content":"
1
2
3
4
5
6
7
#[serde(tag = "filterType")]
#[serde(untagged)]
#[serde(rename = "PRICE_FILTER")]
#[serde(rename_all = "camelCase")]

#[serde(with = "string_or_float")]
pub stop_price: f64,
","site":{"data":{}},"excerpt":"","more":"
1
2
3
4
5
6
7
#[serde(tag = "filterType")]
#[serde(untagged)]
#[serde(rename = "PRICE_FILTER")]
#[serde(rename_all = "camelCase")]

#[serde(with = "string_or_float")]
pub stop_price: f64,
"},{"title":"geth state prune","date":"2023-03-25T11:29:43.000Z","_content":"\n> **_NOTE:_** Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.\n\n## introduction\nA snap-sync'd Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.\n\n## how pruning works\nPruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn't part of the target state trie or genesis state.\n\nGeth prunes the database in three stages:\n\n1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.\n2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.\n3. Compacting database: Geth tidies up the new database to reclaim free space.\n\n## Pruning command\n```\ngeth snapshot prune-state\n```\n\n## references\n- [geth doc](https://geth.ethereum.org/docs/fundamentals/pruning)","source":"_posts/geth/tech_docs/geth.prune.md","raw":"---\ntitle: geth state prune\ndate: 2023-03-25 19:29:43\ntags: [geth]\n---\n\n> **_NOTE:_** Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.\n\n## introduction\nA snap-sync'd Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.\n\n## how pruning works\nPruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn't part of the target state trie or genesis state.\n\nGeth prunes the database in three stages:\n\n1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.\n2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.\n3. Compacting database: Geth tidies up the new database to reclaim free space.\n\n## Pruning command\n```\ngeth snapshot prune-state\n```\n\n## references\n- [geth doc](https://geth.ethereum.org/docs/fundamentals/pruning)","slug":"geth/tech_docs/geth.prune","published":1,"updated":"2023-06-21T09:51:43.628Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzu001xofsj4ipzhfxp","content":"
\n

NOTE: Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.

\n
\n

introduction

A snap-sync’d Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.

\n

how pruning works

Pruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn’t part of the target state trie or genesis state.

\n

Geth prunes the database in three stages:

\n
    \n
  1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.
    2. \n
  2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.
    3. \n
  3. Compacting database: Geth tidies up the new database to reclaim free space.
    4. \n
\n

Pruning command

1
geth snapshot prune-state
\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"
\n

NOTE: Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.

\n
\n

introduction

A snap-sync’d Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.

\n

how pruning works

Pruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn’t part of the target state trie or genesis state.

\n

Geth prunes the database in three stages:

\n
    \n
  1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.
    2. \n
  2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.
    3. \n
  3. Compacting database: Geth tidies up the new database to reclaim free space.
    4. \n
\n

Pruning command

1
geth snapshot prune-state
\n\n

references

\n"},{"title":"geth sync","date":"2023-03-18T08:29:43.000Z","_content":"\n## state \nEthereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we'd need to hash all that data from scratch (which is not efficient).\n\nInstead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).\n\n## state storage\nMPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there's an extra multiplier from there. The net result is that a single state access is expected to amplify into **25-50** random disk accesses. \n\nOf course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.\n\n## Not all accesses are created equal\nThe Merkle Patricia tree is essential for writes (matain the capability to verify data), but it's an overhead for reads. \nAn Ethereum node accesses state in a few different places:\n- When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes. \n- When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).\n- When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.\n\nif we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster. \n\n## snapshot\nGeth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.\nsnapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n). \n\n## devil's in the details\nto maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there's a mini reorg however (even a single block), we're in trouble, because there's no undo. \nTo overcome this limitation, Geth's snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.\nWhenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.\nOf course, there are lots and lots of gotchas and caveats.\n- On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.\n- Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.\n- Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don't cause disk hits.\n- Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there's a chance for an item to be in the diffs, or if we can go to disk immediately.\n- The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.\n\nThe snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap [sync algorithm](https://github.com/ethereum/devp2p/pull/145).\n\n## Consensus layer syncing\nall consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. **Geth cannot sync without being connected to a consensus client.** \nThere are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:\n\n### optimistic sync\nOptimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.\n[more details](https://github.com/ethereum/consensus-specs/blob/dev/sync/optimistic.md)\n\n### checkpoint sync\nAlternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.\n\n## archive nodes\nAn archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node's own storage. \n\nIt is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with `--syncmode snap --gcmode archive`.\n\n\n## light nodes\nA light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. **Light nodes are not currently working on proof-of-stake Ethereum.**\n\n\n\n## full node\n### full\nA full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.\n\n### snap sync (default)\nSnap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state \"on-the-fly\". The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.\n![sync mode](/images/geth/sync.mode.jpg)\n\nSnap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.\n\n\nThe state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don't really understand why regenrated state could be invalidated). This means it is also necessary to have a 'healing' phase where errors in the state are fixed. Geth regularly reports `Syncing, state heal in progress` during state healing - this informs the user that state heal has not finished.\n\nThe healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.\n\nTo summarize, snap sync progresses in the following sequence:\n\n- download and verify headers\n- download block bodies and receipts. In parallel, download raw state data and build state trie\n- heal state trie to account for newly arriving data\n\nA node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.\n\n\n\n\n\n\n\n\n\n# references\n- [geth doc on sync mode](https://geth.ethereum.org/docs/fundamentals/sync-modes)\n- [eth.org blog on snapshot acceleration](https://blog.ethereum.org/2020/07/17/ask-about-geth-snapshot-acceleration)","source":"_posts/geth/tech_docs/geth.sync.mode.md","raw":"---\ntitle: geth sync\ndate: 2023-03-18 16:29:43\ntags: [geth]\n---\n\n## state \nEthereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we'd need to hash all that data from scratch (which is not efficient).\n\nInstead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).\n\n## state storage\nMPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there's an extra multiplier from there. The net result is that a single state access is expected to amplify into **25-50** random disk accesses. \n\nOf course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.\n\n## Not all accesses are created equal\nThe Merkle Patricia tree is essential for writes (matain the capability to verify data), but it's an overhead for reads. \nAn Ethereum node accesses state in a few different places:\n- When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes. \n- When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).\n- When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.\n\nif we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster. \n\n## snapshot\nGeth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.\nsnapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n). \n\n## devil's in the details\nto maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there's a mini reorg however (even a single block), we're in trouble, because there's no undo. \nTo overcome this limitation, Geth's snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.\nWhenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.\nOf course, there are lots and lots of gotchas and caveats.\n- On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.\n- Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.\n- Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don't cause disk hits.\n- Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there's a chance for an item to be in the diffs, or if we can go to disk immediately.\n- The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.\n\nThe snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap [sync algorithm](https://github.com/ethereum/devp2p/pull/145).\n\n## Consensus layer syncing\nall consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. **Geth cannot sync without being connected to a consensus client.** \nThere are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:\n\n### optimistic sync\nOptimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.\n[more details](https://github.com/ethereum/consensus-specs/blob/dev/sync/optimistic.md)\n\n### checkpoint sync\nAlternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.\n\n## archive nodes\nAn archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node's own storage. \n\nIt is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with `--syncmode snap --gcmode archive`.\n\n\n## light nodes\nA light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. **Light nodes are not currently working on proof-of-stake Ethereum.**\n\n\n\n## full node\n### full\nA full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.\n\n### snap sync (default)\nSnap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state \"on-the-fly\". The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.\n![sync mode](/images/geth/sync.mode.jpg)\n\nSnap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.\n\n\nThe state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don't really understand why regenrated state could be invalidated). This means it is also necessary to have a 'healing' phase where errors in the state are fixed. Geth regularly reports `Syncing, state heal in progress` during state healing - this informs the user that state heal has not finished.\n\nThe healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.\n\nTo summarize, snap sync progresses in the following sequence:\n\n- download and verify headers\n- download block bodies and receipts. In parallel, download raw state data and build state trie\n- heal state trie to account for newly arriving data\n\nA node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.\n\n\n\n\n\n\n\n\n\n# references\n- [geth doc on sync mode](https://geth.ethereum.org/docs/fundamentals/sync-modes)\n- [eth.org blog on snapshot acceleration](https://blog.ethereum.org/2020/07/17/ask-about-geth-snapshot-acceleration)","slug":"geth/tech_docs/geth.sync.mode","published":1,"updated":"2023-06-21T01:03:40.833Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzu001zofsj6bkj0wad","content":"

state

Ethereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we’d need to hash all that data from scratch (which is not efficient).

\n

Instead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).

\n

state storage

MPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there’s an extra multiplier from there. The net result is that a single state access is expected to amplify into 25-50 random disk accesses.

\n

Of course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.

\n

Not all accesses are created equal

The Merkle Patricia tree is essential for writes (matain the capability to verify data), but it’s an overhead for reads.
An Ethereum node accesses state in a few different places:

\n
    \n
  • When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes.
    • \n
  • When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).
    • \n
  • When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.
    • \n
\n

if we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster.

\n

snapshot

Geth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.
snapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n).

\n

devil’s in the details

to maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there’s a mini reorg however (even a single block), we’re in trouble, because there’s no undo.
To overcome this limitation, Geth’s snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.
Whenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.
Of course, there are lots and lots of gotchas and caveats.

\n
    \n
  • On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.
    • \n
  • Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.
    • \n
  • Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don’t cause disk hits.
    • \n
  • Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there’s a chance for an item to be in the diffs, or if we can go to disk immediately.
    • \n
  • The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.
    • \n
\n

The snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap sync algorithm.

\n

Consensus layer syncing

all consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. Geth cannot sync without being connected to a consensus client.
There are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:

\n

optimistic sync

Optimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.
more details

\n

checkpoint sync

Alternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.

\n

archive nodes

An archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node’s own storage.

\n

It is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with --syncmode snap --gcmode archive.

\n

light nodes

A light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. Light nodes are not currently working on proof-of-stake Ethereum.

\n

full node

full

A full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.

\n

snap sync (default)

Snap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state “on-the-fly”. The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.
\"sync

\n

Snap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.

\n

The state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don’t really understand why regenrated state could be invalidated). This means it is also necessary to have a ‘healing’ phase where errors in the state are fixed. Geth regularly reports Syncing, state heal in progress during state healing - this informs the user that state heal has not finished.

\n

The healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.

\n

To summarize, snap sync progresses in the following sequence:

\n
    \n
  • download and verify headers
    • \n
  • download block bodies and receipts. In parallel, download raw state data and build state trie
    • \n
  • heal state trie to account for newly arriving data
    • \n
\n

A node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

state

Ethereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we’d need to hash all that data from scratch (which is not efficient).

\n

Instead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).

\n

state storage

MPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there’s an extra multiplier from there. The net result is that a single state access is expected to amplify into 25-50 random disk accesses.

\n

Of course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.

\n

Not all accesses are created equal

The Merkle Patricia tree is essential for writes (matain the capability to verify data), but it’s an overhead for reads.
An Ethereum node accesses state in a few different places:

\n
    \n
  • When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes.
    • \n
  • When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).
    • \n
  • When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.
    • \n
\n

if we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster.

\n

snapshot

Geth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.
snapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n).

\n

devil’s in the details

to maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there’s a mini reorg however (even a single block), we’re in trouble, because there’s no undo.
To overcome this limitation, Geth’s snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.
Whenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.
Of course, there are lots and lots of gotchas and caveats.

\n
    \n
  • On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.
    • \n
  • Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.
    • \n
  • Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don’t cause disk hits.
    • \n
  • Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there’s a chance for an item to be in the diffs, or if we can go to disk immediately.
    • \n
  • The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.
    • \n
\n

The snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap sync algorithm.

\n

Consensus layer syncing

all consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. Geth cannot sync without being connected to a consensus client.
There are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:

\n

optimistic sync

Optimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.
more details

\n

checkpoint sync

Alternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.

\n

archive nodes

An archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node’s own storage.

\n

It is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with --syncmode snap --gcmode archive.

\n

light nodes

A light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. Light nodes are not currently working on proof-of-stake Ethereum.

\n

full node

full

A full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.

\n

snap sync (default)

Snap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state “on-the-fly”. The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.
\"sync

\n

Snap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.

\n

The state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don’t really understand why regenrated state could be invalidated). This means it is also necessary to have a ‘healing’ phase where errors in the state are fixed. Geth regularly reports Syncing, state heal in progress during state healing - this informs the user that state heal has not finished.

\n

The healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.

\n

To summarize, snap sync progresses in the following sequence:

\n
    \n
  • download and verify headers
    • \n
  • download block bodies and receipts. In parallel, download raw state data and build state trie
    • \n
  • heal state trie to account for newly arriving data
    • \n
\n

A node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.

\n

references

\n"},{"title":"geth v1.10.0 summary","date":"2023-03-15T08:29:43.000Z","_content":"\n## introduction\ngeth v1.10.0 has been [released](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.0) on Mar 4 2021. this is a late summary of v1.10.0.\n\n## snapshots\nthe snapshot feature reduces the cost of accessing an account from `O(logN)` to `O(1)`. Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for `O(logN)` disk access on writes. \nProblems it solves\n- **DoS** In 2016, Ethereum sustained its worse DoS attack ever - The [Shanghai Attacks](https://2017.edcon.io/ppt/one/Martin%20Holst%20Swende_The%20%27Shanghai%20%27Attacks_EDCON.pdf) - that lasted about 2-3 months. The attack revolved around bloating Ethereum's state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.\n- **Call** Checking a smart contract's state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.\n- **Sync** There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only **96GB** of data off disk to get a new node joined into the network.\n\ndrawbacks of snapshot\n- A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie. \nuser can disable snapshot via `--snapshot=false`\n\n## snap sync\nWhen Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync). \n- **full sync** minimized trust, choosing to execute all transactions from genesis to head. \n- **fast sync** chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it's ok to download the state associated with `HEAD-64`\n\n### delays of fast sync\n- network latency (download node)\n- io latency (level db random disk access)\n- upload latency (requst with node `hash` to remote servers)\n\nThe core idea of `snap sync` is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:\n- Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.\n- Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.\n- Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO\n\n## offline pruning\nWhen processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could \"just delete\" state data that's old enough to not run the risk of a reorg. it's exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.\nIf you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run `geth snapshot prune-state`.\n\n## transaction unindexing\nNode operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It's also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.\nGeth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use `--txlookuplimit` to control the indexing block range\n\n## preimage discarding\nEthereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.\nthe preimage is the actual key related to the hash. The preimages aren't particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you'll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there's no mechanism to actively delete already stored preimages.\nIf you are using your Geth instance to debug transactions, you can retain the original behavior via `--cache.preimages`. \n\n## ETH/66 protocol\nThe eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.\n\n## chainid enforcement\nGeth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via --rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.\n\n## Database introspection\nEvery now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.\n\n## Unclean shutdown tracking\nFairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it's local chain to the point where it last saved the progress.\n\nGeth v1.10.0 will start tracking and reporting node crashes. We're hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.\n```\nWARN [03-03|06:36:38.734] Unclean shutdown detected booted=2021-02-03T06:47:28+0000 age=3w6d23h\n```\n\n## references\n- [eth foundation blog]()","source":"_posts/geth/tech_docs/geth.v1.10.0.md","raw":"---\ntitle: geth v1.10.0 summary\ndate: 2023-03-15 16:29:43\ntags: [geth]\n---\n\n## introduction\ngeth v1.10.0 has been [released](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.0) on Mar 4 2021. this is a late summary of v1.10.0.\n\n## snapshots\nthe snapshot feature reduces the cost of accessing an account from `O(logN)` to `O(1)`. Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for `O(logN)` disk access on writes. \nProblems it solves\n- **DoS** In 2016, Ethereum sustained its worse DoS attack ever - The [Shanghai Attacks](https://2017.edcon.io/ppt/one/Martin%20Holst%20Swende_The%20%27Shanghai%20%27Attacks_EDCON.pdf) - that lasted about 2-3 months. The attack revolved around bloating Ethereum's state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.\n- **Call** Checking a smart contract's state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.\n- **Sync** There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only **96GB** of data off disk to get a new node joined into the network.\n\ndrawbacks of snapshot\n- A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie. \nuser can disable snapshot via `--snapshot=false`\n\n## snap sync\nWhen Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync). \n- **full sync** minimized trust, choosing to execute all transactions from genesis to head. \n- **fast sync** chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it's ok to download the state associated with `HEAD-64`\n\n### delays of fast sync\n- network latency (download node)\n- io latency (level db random disk access)\n- upload latency (requst with node `hash` to remote servers)\n\nThe core idea of `snap sync` is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:\n- Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.\n- Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.\n- Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO\n\n## offline pruning\nWhen processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could \"just delete\" state data that's old enough to not run the risk of a reorg. it's exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.\nIf you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run `geth snapshot prune-state`.\n\n## transaction unindexing\nNode operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It's also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.\nGeth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use `--txlookuplimit` to control the indexing block range\n\n## preimage discarding\nEthereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.\nthe preimage is the actual key related to the hash. The preimages aren't particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you'll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there's no mechanism to actively delete already stored preimages.\nIf you are using your Geth instance to debug transactions, you can retain the original behavior via `--cache.preimages`. \n\n## ETH/66 protocol\nThe eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.\n\n## chainid enforcement\nGeth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via --rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.\n\n## Database introspection\nEvery now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.\n\n## Unclean shutdown tracking\nFairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it's local chain to the point where it last saved the progress.\n\nGeth v1.10.0 will start tracking and reporting node crashes. We're hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.\n```\nWARN [03-03|06:36:38.734] Unclean shutdown detected booted=2021-02-03T06:47:28+0000 age=3w6d23h\n```\n\n## references\n- [eth foundation blog]()","slug":"geth/tech_docs/geth.v1.10.0","published":1,"updated":"2023-06-18T15:06:29.245Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzv0022ofsjde824or5","content":"

introduction

geth v1.10.0 has been released on Mar 4 2021. this is a late summary of v1.10.0.

\n

snapshots

the snapshot feature reduces the cost of accessing an account from O(logN) to O(1). Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for O(logN) disk access on writes.
Problems it solves

\n
    \n
  • DoS In 2016, Ethereum sustained its worse DoS attack ever - The Shanghai Attacks - that lasted about 2-3 months. The attack revolved around bloating Ethereum’s state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.
    • \n
  • Call Checking a smart contract’s state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.
    • \n
  • Sync There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only 96GB of data off disk to get a new node joined into the network.
    • \n
\n

drawbacks of snapshot

\n
    \n
  • A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie.
    user can disable snapshot via --snapshot=false
    • \n
\n

snap sync

When Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync).

\n
    \n
  • full sync minimized trust, choosing to execute all transactions from genesis to head.
    • \n
  • fast sync chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it’s ok to download the state associated with HEAD-64
    • \n
\n

delays of fast sync

    \n
  • network latency (download node)
    • \n
  • io latency (level db random disk access)
    • \n
  • upload latency (requst with node hash to remote servers)
    • \n
\n

The core idea of snap sync is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:

\n
    \n
  • Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.
    • \n
  • Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.
    • \n
  • Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO
    • \n
\n

offline pruning

When processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could “just delete” state data that’s old enough to not run the risk of a reorg. it’s exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.
If you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run geth snapshot prune-state.

\n

transaction unindexing

Node operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It’s also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.
Geth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use --txlookuplimit to control the indexing block range

\n

preimage discarding

Ethereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.
the preimage is the actual key related to the hash. The preimages aren’t particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you’ll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there’s no mechanism to actively delete already stored preimages.
If you are using your Geth instance to debug transactions, you can retain the original behavior via --cache.preimages.

\n

ETH/66 protocol

The eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.

\n

chainid enforcement

Geth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via –rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.

\n

Database introspection

Every now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.

\n

Unclean shutdown tracking

Fairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it’s local chain to the point where it last saved the progress.

\n

Geth v1.10.0 will start tracking and reporting node crashes. We’re hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.

\n
1
WARN [03-03|06:36:38.734] Unclean shutdown detected        booted=2021-02-03T06:47:28+0000 age=3w6d23h
\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

geth v1.10.0 has been released on Mar 4 2021. this is a late summary of v1.10.0.

\n

snapshots

the snapshot feature reduces the cost of accessing an account from O(logN) to O(1). Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for O(logN) disk access on writes.
Problems it solves

\n
    \n
  • DoS In 2016, Ethereum sustained its worse DoS attack ever - The Shanghai Attacks - that lasted about 2-3 months. The attack revolved around bloating Ethereum’s state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.
    • \n
  • Call Checking a smart contract’s state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.
    • \n
  • Sync There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only 96GB of data off disk to get a new node joined into the network.
    • \n
\n

drawbacks of snapshot

\n
    \n
  • A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie.
    user can disable snapshot via --snapshot=false
    • \n
\n

snap sync

When Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync).

\n
    \n
  • full sync minimized trust, choosing to execute all transactions from genesis to head.
    • \n
  • fast sync chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it’s ok to download the state associated with HEAD-64
    • \n
\n

delays of fast sync

    \n
  • network latency (download node)
    • \n
  • io latency (level db random disk access)
    • \n
  • upload latency (requst with node hash to remote servers)
    • \n
\n

The core idea of snap sync is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:

\n
    \n
  • Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.
    • \n
  • Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.
    • \n
  • Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO
    • \n
\n

offline pruning

When processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could “just delete” state data that’s old enough to not run the risk of a reorg. it’s exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.
If you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run geth snapshot prune-state.

\n

transaction unindexing

Node operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It’s also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.
Geth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use --txlookuplimit to control the indexing block range

\n

preimage discarding

Ethereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.
the preimage is the actual key related to the hash. The preimages aren’t particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you’ll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there’s no mechanism to actively delete already stored preimages.
If you are using your Geth instance to debug transactions, you can retain the original behavior via --cache.preimages.

\n

ETH/66 protocol

The eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.

\n

chainid enforcement

Geth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via –rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.

\n

Database introspection

Every now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.

\n

Unclean shutdown tracking

Fairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it’s local chain to the point where it last saved the progress.

\n

Geth v1.10.0 will start tracking and reporting node crashes. We’re hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.

\n
1
WARN [03-03|06:36:38.734] Unclean shutdown detected        booted=2021-02-03T06:47:28+0000 age=3w6d23h
\n\n

references

\n"},{"title":"rust std smart pointer & interior mutability","date":"2023-06-03T14:04:38.000Z","_content":"# smart pointer\n## [Rc](https://doc.rust-lang.org/std/rc/struct.Rc.html)\nA single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.\n\n\n\n# internal mutibility\n## [Cell](https://doc.rust-lang.org/stable/std/cell/struct.Cell.html)\n`Cell` enables mutation inside an immutable value. In other words, it enables `interior mutability`. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.\n### methods\n- `fn get(&self) -> T`\n- `fn set(&self, val: T)`\n- `fn swap(&self, other: &Cell)`\n- `fn replace(&self, val: T) -> T`\nReplaces the contained value with val, and returns the old contained value\n- `fn into_inner(self) -> T`\n- `const fn as_ptr(&self) -> *mut T`\n- `fn get_mut(&mut self) -> &mut T`\n- `fn from_mut(t: &mut T) -> &Cell`\n\n### traits\n```rust\nimpl !Sync for Cell // cannot be used in other threads\n```\n\n## [OnceCell](https://doc.rust-lang.org/stable/std/cell/struct.OnceCell.html)\nA cell which can be written to only once.\n### special methods\n- `fn get_or_init(&self, f: F) -> &T`\n\n## [LazyCell](https://doc.rust-lang.org/stable/std/cell/struct.LazyCell.html)\nA value which is initialized on the first access\n\n## [UnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.UnsafeCell.html#)\n`UnsafeCell` opts-out of the immutability guarantee for `&T`: a shared reference `&UnsafeCell` may point to data that is being mutated. This is called `interior mutability`.\nAll other types that allow internal mutability, such as `Cell` and `RefCell`, internally use `UnsafeCell` to wrap their data.\nNote that only the immutability guarantee for shared references is affected by `UnsafeCell`. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference). \n\n### methods\n- `pub const fn get(&self) -> *mut T`\nGets a mutable pointer to the wrapped value.\n- `pub fn get_mut(&mut self) -> &mut T`\nReturns a mutable reference to the underlying data\n- `pub const fn raw_get(this: *const UnsafeCell) -> *mut T`\nGets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:\n```rust\nuse std::cell::UnsafeCell;\nuse std::mem::MaybeUninit;\n\nlet m = MaybeUninit::>::uninit();\nunsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }\nlet uc = unsafe { m.assume_init() };\n\nassert_eq!(uc.into_inner(), 5);\n```\n- `fn into_inner(self) -> T`\nUnwraps the value, consuming the cell.\n\n## [SyncUnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.SyncUnsafeCell.html)\nThis is just an `UnsafeCell`, except it implements `Sync` if T implements Sync.\n\n## [std::cell::RefCell](https://doc.rust-lang.org/stable/std/cell/struct.RefCell.html)\nA mutable memory location with **dynamically** checked borrow rules\n- `fn borrow(&self) -> Ref<'_, T>`\n- `fn borrow_mut(&self) -> RefMut<'_, T>`\n- `fn as_ptr(&self) -> *mut T`\n\n# borrow\n## [std::borrow::Cow](https://doc.rust-lang.org/std/borrow/enum.Cow.html)\n","source":"_posts/rust/rust_std/rust-smart-pointer-and-internal-mutibility.md","raw":"---\ntitle: rust std smart pointer & interior mutability\ndate: 2023-06-03 22:04:38\ntags: [rust-std]\n---\n# smart pointer\n## [Rc](https://doc.rust-lang.org/std/rc/struct.Rc.html)\nA single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.\n\n\n\n# internal mutibility\n## [Cell](https://doc.rust-lang.org/stable/std/cell/struct.Cell.html)\n`Cell` enables mutation inside an immutable value. In other words, it enables `interior mutability`. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.\n### methods\n- `fn get(&self) -> T`\n- `fn set(&self, val: T)`\n- `fn swap(&self, other: &Cell)`\n- `fn replace(&self, val: T) -> T`\nReplaces the contained value with val, and returns the old contained value\n- `fn into_inner(self) -> T`\n- `const fn as_ptr(&self) -> *mut T`\n- `fn get_mut(&mut self) -> &mut T`\n- `fn from_mut(t: &mut T) -> &Cell`\n\n### traits\n```rust\nimpl !Sync for Cell // cannot be used in other threads\n```\n\n## [OnceCell](https://doc.rust-lang.org/stable/std/cell/struct.OnceCell.html)\nA cell which can be written to only once.\n### special methods\n- `fn get_or_init(&self, f: F) -> &T`\n\n## [LazyCell](https://doc.rust-lang.org/stable/std/cell/struct.LazyCell.html)\nA value which is initialized on the first access\n\n## [UnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.UnsafeCell.html#)\n`UnsafeCell` opts-out of the immutability guarantee for `&T`: a shared reference `&UnsafeCell` may point to data that is being mutated. This is called `interior mutability`.\nAll other types that allow internal mutability, such as `Cell` and `RefCell`, internally use `UnsafeCell` to wrap their data.\nNote that only the immutability guarantee for shared references is affected by `UnsafeCell`. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference). \n\n### methods\n- `pub const fn get(&self) -> *mut T`\nGets a mutable pointer to the wrapped value.\n- `pub fn get_mut(&mut self) -> &mut T`\nReturns a mutable reference to the underlying data\n- `pub const fn raw_get(this: *const UnsafeCell) -> *mut T`\nGets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:\n```rust\nuse std::cell::UnsafeCell;\nuse std::mem::MaybeUninit;\n\nlet m = MaybeUninit::>::uninit();\nunsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }\nlet uc = unsafe { m.assume_init() };\n\nassert_eq!(uc.into_inner(), 5);\n```\n- `fn into_inner(self) -> T`\nUnwraps the value, consuming the cell.\n\n## [SyncUnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.SyncUnsafeCell.html)\nThis is just an `UnsafeCell`, except it implements `Sync` if T implements Sync.\n\n## [std::cell::RefCell](https://doc.rust-lang.org/stable/std/cell/struct.RefCell.html)\nA mutable memory location with **dynamically** checked borrow rules\n- `fn borrow(&self) -> Ref<'_, T>`\n- `fn borrow_mut(&self) -> RefMut<'_, T>`\n- `fn as_ptr(&self) -> *mut T`\n\n# borrow\n## [std::borrow::Cow](https://doc.rust-lang.org/std/borrow/enum.Cow.html)\n","slug":"rust/rust_std/rust-smart-pointer-and-internal-mutibility","published":1,"updated":"2023-07-09T15:15:15.385Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzv0024ofsjcdqa4dbl","content":"

smart pointer

Rc

A single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.

\n

internal mutibility

Cell

Cell<T> enables mutation inside an immutable value. In other words, it enables interior mutability. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.

\n

methods

    \n
  • fn get(&self) -> T
    • \n
  • fn set(&self, val: T)
    • \n
  • fn swap(&self, other: &Cell<T>)
    • \n
  • fn replace(&self, val: T) -> T
    Replaces the contained value with val, and returns the old contained value
    • \n
  • fn into_inner(self) -> T
    • \n
  • const fn as_ptr(&self) -> *mut T
    • \n
  • fn get_mut(&mut self) -> &mut T
    • \n
  • fn from_mut(t: &mut T) -> &Cell<T>
    • \n
\n

traits

1
impl<T> !Sync for Cell<T>  // cannot be used in other threads
\n\n

OnceCell

A cell which can be written to only once.

\n

special methods

    \n
  • fn get_or_init<F>(&self, f: F) -> &T
    • \n
\n

LazyCell

A value which is initialized on the first access

\n

UnsafeCell

UnsafeCell<T> opts-out of the immutability guarantee for &T: a shared reference &UnsafeCell<T> may point to data that is being mutated. This is called interior mutability.
All other types that allow internal mutability, such as Cell<T> and RefCell<T>, internally use UnsafeCell to wrap their data.
Note that only the immutability guarantee for shared references is affected by UnsafeCell. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference).

\n

methods

    \n
  • pub const fn get(&self) -> *mut T
    Gets a mutable pointer to the wrapped value.
    • \n
  • pub fn get_mut(&mut self) -> &mut T
    Returns a mutable reference to the underlying data
    • \n
  • pub const fn raw_get(this: *const UnsafeCell<T>) -> *mut T
    Gets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:
    1
    2
    3
    4
    5
    6
    7
    8
    use std::cell::UnsafeCell;
    use std::mem::MaybeUninit;

    let m = MaybeUninit::<UnsafeCell<i32>>::uninit();
    unsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }
    let uc = unsafe { m.assume_init() };

    assert_eq!(uc.into_inner(), 5);
    • \n
  • fn into_inner(self) -> T
    Unwraps the value, consuming the cell.
    • \n
\n

SyncUnsafeCell

This is just an UnsafeCell, except it implements Sync if T implements Sync.

\n

std::cell::RefCell

A mutable memory location with dynamically checked borrow rules

\n
    \n
  • fn borrow(&self) -> Ref<'_, T>
    • \n
  • fn borrow_mut(&self) -> RefMut<'_, T>
    • \n
  • fn as_ptr(&self) -> *mut T
    • \n
\n

borrow

std::borrow::Cow

","site":{"data":{}},"excerpt":"","more":"

smart pointer

Rc

A single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.

\n

internal mutibility

Cell

Cell<T> enables mutation inside an immutable value. In other words, it enables interior mutability. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.

\n

methods

    \n
  • fn get(&self) -> T
    • \n
  • fn set(&self, val: T)
    • \n
  • fn swap(&self, other: &Cell<T>)
    • \n
  • fn replace(&self, val: T) -> T
    Replaces the contained value with val, and returns the old contained value
    • \n
  • fn into_inner(self) -> T
    • \n
  • const fn as_ptr(&self) -> *mut T
    • \n
  • fn get_mut(&mut self) -> &mut T
    • \n
  • fn from_mut(t: &mut T) -> &Cell<T>
    • \n
\n

traits

1
impl<T> !Sync for Cell<T>  // cannot be used in other threads
\n\n

OnceCell

A cell which can be written to only once.

\n

special methods

    \n
  • fn get_or_init<F>(&self, f: F) -> &T
    • \n
\n

LazyCell

A value which is initialized on the first access

\n

UnsafeCell

UnsafeCell<T> opts-out of the immutability guarantee for &T: a shared reference &UnsafeCell<T> may point to data that is being mutated. This is called interior mutability.
All other types that allow internal mutability, such as Cell<T> and RefCell<T>, internally use UnsafeCell to wrap their data.
Note that only the immutability guarantee for shared references is affected by UnsafeCell. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference).

\n

methods

    \n
  • pub const fn get(&self) -> *mut T
    Gets a mutable pointer to the wrapped value.
    • \n
  • pub fn get_mut(&mut self) -> &mut T
    Returns a mutable reference to the underlying data
    • \n
  • pub const fn raw_get(this: *const UnsafeCell<T>) -> *mut T
    Gets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:
    1
    2
    3
    4
    5
    6
    7
    8
    use std::cell::UnsafeCell;
    use std::mem::MaybeUninit;

    let m = MaybeUninit::<UnsafeCell<i32>>::uninit();
    unsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }
    let uc = unsafe { m.assume_init() };

    assert_eq!(uc.into_inner(), 5);
    • \n
  • fn into_inner(self) -> T
    Unwraps the value, consuming the cell.
    • \n
\n

SyncUnsafeCell

This is just an UnsafeCell, except it implements Sync if T implements Sync.

\n

std::cell::RefCell

A mutable memory location with dynamically checked borrow rules

\n
    \n
  • fn borrow(&self) -> Ref<'_, T>
    • \n
  • fn borrow_mut(&self) -> RefMut<'_, T>
    • \n
  • fn as_ptr(&self) -> *mut T
    • \n
\n

borrow

std::borrow::Cow

"},{"title":"rust std data structure (1D)","date":"2023-05-01T14:04:38.000Z","_content":"\n## array\nA **fixed-size** array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.\n```rust\ntodo!()\n```\n\n## slice\nA **dynamically-sized view** into a contiguous sequence, [T].\n- `len()`: Returns the number of elements in the slice\n- `is_empty()`\n- `first()` Returns the first element of the slice, or `None` if it is empty.\n- `first_mut()` Returns a mutable **pointer** to the first element of the slice, or `None` if it is empty\n- `split_first()` Returns the first and all the rest of the elements of the slice, or `None` if it is empty.\n- `split_first_mut()` \n- `split_last()`\n- `split_last_mut()`\n- `last()`\n- `last_mut()`\n- `get(index: I)` Returns a reference to an element or subslice depending on the type of index.\n```rust\nlet v = [10, 40, 30];\nassert_eq!(Some(&40), v.get(1));\nassert_eq!(Some(&[10, 40][..]), v.get(0..2));\n```\n- `get_mut(index: I)`\n- `get_unchecked(index: I)` Returns a reference to an element or subslice, without doing bounds checking\n- `get_unchecked_mut(index: I)`\n- `as_ptr(&self) -> *const T` Returns a raw pointer to the slice's buffer\n```rust\nlet x = &[1, 2, 4];\nlet x_ptr = x.as_ptr();\nunsafe {\n for i in 0..x.len() {\n assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));\n }\n}\n```\n- `as_mut_ptr(&mut self) -> *mut T` \n```rust\nlet x = &mut [1, 2, 4];\nlet x_ptr = x.as_mut_ptr();\nunsafe {\n for i in 0..x.len() {\n *x_ptr.add(i) += 2;\n }\n}\nassert_eq!(x, &[3, 4, 6]);\n```\n- `as_ptr_range(&self) -> Range<*const T>` Returns the two raw pointers spanning the slice.\n```rust\npub const fn as_ptr_range(&self) -> Range<*const T> {\n let start = self.as_ptr();\n let end = unsafe { start.add(self.len()) };\n start..end\n}\n```\n- `as_mut_ptr_range(&mut self) -> Range<*mut T>`\n- `swap(&mut self, a: usize, b: usize)` Swaps two elements in the slice.\n- `reverse(&mut self)` Reverses the order of elements in the slice, in place.\n- `windows(&self, size: usize)` Returns an iterator over all contiguous windows of length `size`. The windows overlap. If the slice is shorter than `size`, the iterator returns no values.\n```rust\nlet slice = ['r', 'u', 's', 't'];\nlet mut iter = slice.windows(2);\nassert_eq!(iter.next().unwrap(), &['r', 'u']);\nassert_eq!(iter.next().unwrap(), &['u', 's']);\nassert_eq!(iter.next().unwrap(), &['s', 't']);\nassert!(iter.next().is_none());\n```\n- `chunks(&self, chunk_size: usize)` Returns an iterator over `chunk_size` elements of the slice at a time\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert_eq!(iter.next().unwrap(), &['m']);\nassert!(iter.next().is_none());\n```\n- `chunks_mut()`\n- `chunks_exact(&self, chunk_size: usize)`\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks_exact(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert!(iter.next().is_none());\nassert_eq!(iter.remainder(), &['m']);\n```\n- `as_chunks_unchecked(&self)` Splits the slice into a slice of `N`-element arrays, assuming that there's no remainder\n- `as_chunks(&self)` Splits the slice into a slice of `N`-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than `N`\n- `as_rchunks(&self)` r means reverse\n- `group_by(&self, pred: F)` Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on `slice[0]` and `slice[1]` then on `slice[1]` and `slice[2]` and so on\n```rust\n#![feature(slice_group_by)]\nlet slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];\nlet mut iter = slice.group_by(|a, b| a <= b);\nassert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3, 4][..]));\nassert_eq!(iter.next(), None);\n```\n- `split_at(&self, mid: usize)` Divides one slice into two at an index.\n- `split(&self, pred: F)` Returns an iterator over subslices separated by elements that match `pred`. The matched element is not contained in the subslices.\n- `splitn(&self, n: usize, pred: F)` \n- `contains(&self, x: &T)` Returns `true` if the slice contains an element with the given value.\n- `starts_with(&self, needle: &[T])` eturns `true` if `needle` is a prefix of the slice\n```rust\nlet v = [10, 40, 30];\nassert!(v.starts_with(&[10]));\nassert!(v.starts_with(&[10, 40]));\nassert!(!v.starts_with(&[50]));\n```\n- `ends_with(&self, needle: &[T])` \n- `strip_prefix` Returns a subslice with the prefix removed.\n```rust\nlet v = &[10, 40, 30];\nassert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));\nassert_eq!(v.strip_prefix(&[50]), None);\nlet prefix : &str = \"he\";\nassert_eq!(b\"hello\".strip_prefix(prefix.as_bytes()),\n Some(b\"llo\".as_ref()));\n```\n- `strip_suffix`\n- `binary_search(&self, x: &T)` Binary searches this slice for a given element.\n- `sort_unstable(&mut self)` Sorts the slice, but might not preserve the order of equal elements.\n- `rotate_left(&mut self, mid: usize)` Rotates the slice in-place such that the first `mid` elements of the slice move to the end while the last `self.len() - mid` elements move to the front.\n- `fill(&mut self, value: T)` Fills `self` with elements by cloning `value`.\n- `clone_from_slice(&mut self, src: &[T])` Copies the elements from `src` into `self`.\n- `copy_from_slice(&mut self, src: &[T])` \n- `is_sorted(&self)` \n- `take<'a, R: OneSidedRange>(self: &mut &'a Self, range: R)` Removes the subslice corresponding to the given range\n- `get_many_mut` Returns mutable references to many indices at once.\n```rust\n#![feature(get_many_mut)]\nlet v = &mut [1, 2, 3];\nif let Ok([a, b]) = v.get_many_mut([0, 2]) {\n *a = 413;\n *b = 612;\n}\nassert_eq!(v, &[413, 2, 612]);\n```\n\n## alloc::vec::Vec\n- `fn truncate(&mut self, len: usize)` Shortens the vector, keeping the first `len` elements and dropping the rest\n\n## std::collections::VecDeque\nA double-ended queue (deque) implemented with a growable ring buffer.\nSince VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.\n\n- `swap(&mut self, i: usize, j: usize)`\n- `reserve_exact(&mut self, additional: usize)` Reserves the minimum capacity for at least `additional` more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.\n- `reserve(&mut self, additional: usize)`\n- `shrink_to_fit(&mut self)` Shrinks the capacity of the deque as much as possible.\n- `truncate(&mut self, len: usize)` Shortens the deque, keeping the first `len` elements and dropping the rest.\n```rust\nuse std::collections::VecDeque;\nlet mut buf = VecDeque::new();\nbuf.push_back(5);\nbuf.push_back(10);\nbuf.push_back(15);\nassert_eq!(buf, [5, 10, 15]);\nbuf.truncate(1);\nassert_eq!(buf, [5]);\n```\n- `iter(&self)`\n- `as_slices(&self)`\n- `slice_ranges(&self, range: R)` Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range\n- `range(&self, range: R)` Creates an iterator that covers the specified range in the deque.\n```rust\nuse std::collections::VecDeque;\nlet deque: VecDeque<_> = [1, 2, 3].into();\nlet range = deque.range(2..).copied().collect::>();\nassert_eq!(range, [3]);\n// A full range covers all contents\nlet all = deque.range(..);\nassert_eq!(all.len(), 3);\n```\n- `drain(&mut self, range: R)` Removes the specified range from the deque in bulk, returning all removed elements as an iterator.\n- `clear(&mut self)`\n- `contains(&self, x: &T)` Returns `true` if the deque contains an element equal to the given value\n- `front(&self)` Provides a reference to the front element\n- `front_mut(&mut self)`\n- `back(&self)`\n- `back_mut(&mut self)`\n- `pop_front(&mut self)`\n- `pop_back(&mut self)`\n- `push_front(&mut self, value: T)`\n- `push_back(&mut self, value: T)`\n\n## [std::collections::LinkedList](https://doc.rust-lang.org/std/collections/struct.LinkedList.html)","source":"_posts/rust/rust_std/rust-std-data-structure-1.md","raw":"---\ntitle: rust std data structure (1D)\ndate: 2023-05-01 22:04:38\ntags: [rust-std]\n---\n\n## array\nA **fixed-size** array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.\n```rust\ntodo!()\n```\n\n## slice\nA **dynamically-sized view** into a contiguous sequence, [T].\n- `len()`: Returns the number of elements in the slice\n- `is_empty()`\n- `first()` Returns the first element of the slice, or `None` if it is empty.\n- `first_mut()` Returns a mutable **pointer** to the first element of the slice, or `None` if it is empty\n- `split_first()` Returns the first and all the rest of the elements of the slice, or `None` if it is empty.\n- `split_first_mut()` \n- `split_last()`\n- `split_last_mut()`\n- `last()`\n- `last_mut()`\n- `get(index: I)` Returns a reference to an element or subslice depending on the type of index.\n```rust\nlet v = [10, 40, 30];\nassert_eq!(Some(&40), v.get(1));\nassert_eq!(Some(&[10, 40][..]), v.get(0..2));\n```\n- `get_mut(index: I)`\n- `get_unchecked(index: I)` Returns a reference to an element or subslice, without doing bounds checking\n- `get_unchecked_mut(index: I)`\n- `as_ptr(&self) -> *const T` Returns a raw pointer to the slice's buffer\n```rust\nlet x = &[1, 2, 4];\nlet x_ptr = x.as_ptr();\nunsafe {\n for i in 0..x.len() {\n assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));\n }\n}\n```\n- `as_mut_ptr(&mut self) -> *mut T` \n```rust\nlet x = &mut [1, 2, 4];\nlet x_ptr = x.as_mut_ptr();\nunsafe {\n for i in 0..x.len() {\n *x_ptr.add(i) += 2;\n }\n}\nassert_eq!(x, &[3, 4, 6]);\n```\n- `as_ptr_range(&self) -> Range<*const T>` Returns the two raw pointers spanning the slice.\n```rust\npub const fn as_ptr_range(&self) -> Range<*const T> {\n let start = self.as_ptr();\n let end = unsafe { start.add(self.len()) };\n start..end\n}\n```\n- `as_mut_ptr_range(&mut self) -> Range<*mut T>`\n- `swap(&mut self, a: usize, b: usize)` Swaps two elements in the slice.\n- `reverse(&mut self)` Reverses the order of elements in the slice, in place.\n- `windows(&self, size: usize)` Returns an iterator over all contiguous windows of length `size`. The windows overlap. If the slice is shorter than `size`, the iterator returns no values.\n```rust\nlet slice = ['r', 'u', 's', 't'];\nlet mut iter = slice.windows(2);\nassert_eq!(iter.next().unwrap(), &['r', 'u']);\nassert_eq!(iter.next().unwrap(), &['u', 's']);\nassert_eq!(iter.next().unwrap(), &['s', 't']);\nassert!(iter.next().is_none());\n```\n- `chunks(&self, chunk_size: usize)` Returns an iterator over `chunk_size` elements of the slice at a time\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert_eq!(iter.next().unwrap(), &['m']);\nassert!(iter.next().is_none());\n```\n- `chunks_mut()`\n- `chunks_exact(&self, chunk_size: usize)`\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks_exact(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert!(iter.next().is_none());\nassert_eq!(iter.remainder(), &['m']);\n```\n- `as_chunks_unchecked(&self)` Splits the slice into a slice of `N`-element arrays, assuming that there's no remainder\n- `as_chunks(&self)` Splits the slice into a slice of `N`-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than `N`\n- `as_rchunks(&self)` r means reverse\n- `group_by(&self, pred: F)` Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on `slice[0]` and `slice[1]` then on `slice[1]` and `slice[2]` and so on\n```rust\n#![feature(slice_group_by)]\nlet slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];\nlet mut iter = slice.group_by(|a, b| a <= b);\nassert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3, 4][..]));\nassert_eq!(iter.next(), None);\n```\n- `split_at(&self, mid: usize)` Divides one slice into two at an index.\n- `split(&self, pred: F)` Returns an iterator over subslices separated by elements that match `pred`. The matched element is not contained in the subslices.\n- `splitn(&self, n: usize, pred: F)` \n- `contains(&self, x: &T)` Returns `true` if the slice contains an element with the given value.\n- `starts_with(&self, needle: &[T])` eturns `true` if `needle` is a prefix of the slice\n```rust\nlet v = [10, 40, 30];\nassert!(v.starts_with(&[10]));\nassert!(v.starts_with(&[10, 40]));\nassert!(!v.starts_with(&[50]));\n```\n- `ends_with(&self, needle: &[T])` \n- `strip_prefix` Returns a subslice with the prefix removed.\n```rust\nlet v = &[10, 40, 30];\nassert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));\nassert_eq!(v.strip_prefix(&[50]), None);\nlet prefix : &str = \"he\";\nassert_eq!(b\"hello\".strip_prefix(prefix.as_bytes()),\n Some(b\"llo\".as_ref()));\n```\n- `strip_suffix`\n- `binary_search(&self, x: &T)` Binary searches this slice for a given element.\n- `sort_unstable(&mut self)` Sorts the slice, but might not preserve the order of equal elements.\n- `rotate_left(&mut self, mid: usize)` Rotates the slice in-place such that the first `mid` elements of the slice move to the end while the last `self.len() - mid` elements move to the front.\n- `fill(&mut self, value: T)` Fills `self` with elements by cloning `value`.\n- `clone_from_slice(&mut self, src: &[T])` Copies the elements from `src` into `self`.\n- `copy_from_slice(&mut self, src: &[T])` \n- `is_sorted(&self)` \n- `take<'a, R: OneSidedRange>(self: &mut &'a Self, range: R)` Removes the subslice corresponding to the given range\n- `get_many_mut` Returns mutable references to many indices at once.\n```rust\n#![feature(get_many_mut)]\nlet v = &mut [1, 2, 3];\nif let Ok([a, b]) = v.get_many_mut([0, 2]) {\n *a = 413;\n *b = 612;\n}\nassert_eq!(v, &[413, 2, 612]);\n```\n\n## alloc::vec::Vec\n- `fn truncate(&mut self, len: usize)` Shortens the vector, keeping the first `len` elements and dropping the rest\n\n## std::collections::VecDeque\nA double-ended queue (deque) implemented with a growable ring buffer.\nSince VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.\n\n- `swap(&mut self, i: usize, j: usize)`\n- `reserve_exact(&mut self, additional: usize)` Reserves the minimum capacity for at least `additional` more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.\n- `reserve(&mut self, additional: usize)`\n- `shrink_to_fit(&mut self)` Shrinks the capacity of the deque as much as possible.\n- `truncate(&mut self, len: usize)` Shortens the deque, keeping the first `len` elements and dropping the rest.\n```rust\nuse std::collections::VecDeque;\nlet mut buf = VecDeque::new();\nbuf.push_back(5);\nbuf.push_back(10);\nbuf.push_back(15);\nassert_eq!(buf, [5, 10, 15]);\nbuf.truncate(1);\nassert_eq!(buf, [5]);\n```\n- `iter(&self)`\n- `as_slices(&self)`\n- `slice_ranges(&self, range: R)` Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range\n- `range(&self, range: R)` Creates an iterator that covers the specified range in the deque.\n```rust\nuse std::collections::VecDeque;\nlet deque: VecDeque<_> = [1, 2, 3].into();\nlet range = deque.range(2..).copied().collect::>();\nassert_eq!(range, [3]);\n// A full range covers all contents\nlet all = deque.range(..);\nassert_eq!(all.len(), 3);\n```\n- `drain(&mut self, range: R)` Removes the specified range from the deque in bulk, returning all removed elements as an iterator.\n- `clear(&mut self)`\n- `contains(&self, x: &T)` Returns `true` if the deque contains an element equal to the given value\n- `front(&self)` Provides a reference to the front element\n- `front_mut(&mut self)`\n- `back(&self)`\n- `back_mut(&mut self)`\n- `pop_front(&mut self)`\n- `pop_back(&mut self)`\n- `push_front(&mut self, value: T)`\n- `push_back(&mut self, value: T)`\n\n## [std::collections::LinkedList](https://doc.rust-lang.org/std/collections/struct.LinkedList.html)","slug":"rust/rust_std/rust-std-data-structure-1","published":1,"updated":"2023-07-11T10:18:40.048Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzw0027ofsj952sej9f","content":"

array

A fixed-size array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.

\n
1
todo!()
\n\n

slice

A dynamically-sized view into a contiguous sequence, [T].

\n
    \n
  • len(): Returns the number of elements in the slice
    • \n
  • is_empty()
    • \n
  • first() Returns the first element of the slice, or None if it is empty.
    • \n
  • first_mut() Returns a mutable pointer to the first element of the slice, or None if it is empty
    • \n
  • split_first() Returns the first and all the rest of the elements of the slice, or None if it is empty.
    • \n
  • split_first_mut()
    • \n
  • split_last()
    • \n
  • split_last_mut()
    • \n
  • last()
    • \n
  • last_mut()
    • \n
  • get<I>(index: I) Returns a reference to an element or subslice depending on the type of index.
    1
    2
    3
    let v = [10, 40, 30];
    assert_eq!(Some(&40), v.get(1));
    assert_eq!(Some(&[10, 40][..]), v.get(0..2));
    • \n
  • get_mut<I>(index: I)
    • \n
  • get_unchecked<I>(index: I) Returns a reference to an element or subslice, without doing bounds checking
    • \n
  • get_unchecked_mut<I>(index: I)
    • \n
  • as_ptr(&self) -> *const T Returns a raw pointer to the slice’s buffer
    1
    2
    3
    4
    5
    6
    7
    let x = &[1, 2, 4];
    let x_ptr = x.as_ptr();
    unsafe {
        for i in 0..x.len() {
            assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));
        }
    }
    • \n
  • as_mut_ptr(&mut self) -> *mut T 
    1
    2
    3
    4
    5
    6
    7
    8
    let x = &mut [1, 2, 4];
    let x_ptr = x.as_mut_ptr();
    unsafe {
        for i in 0..x.len() {
            *x_ptr.add(i) += 2;
        }
    }
    assert_eq!(x, &[3, 4, 6]);
    • \n
  • as_ptr_range(&self) -> Range<*const T> Returns the two raw pointers spanning the slice.
    1
    2
    3
    4
    5
    pub const fn as_ptr_range(&self) -> Range<*const T> {
        let start = self.as_ptr();
        let end = unsafe { start.add(self.len()) };
        start..end
    }
    • \n
  • as_mut_ptr_range(&mut self) -> Range<*mut T>
    • \n
  • swap(&mut self, a: usize, b: usize) Swaps two elements in the slice.
    • \n
  • reverse(&mut self) Reverses the order of elements in the slice, in place.
    • \n
  • windows(&self, size: usize) Returns an iterator over all contiguous windows of length size. The windows overlap. If the slice is shorter than size, the iterator returns no values.
    1
    2
    3
    4
    5
    6
    let slice = ['r', 'u', 's', 't'];
    let mut iter = slice.windows(2);
    assert_eq!(iter.next().unwrap(), &['r', 'u']);
    assert_eq!(iter.next().unwrap(), &['u', 's']);
    assert_eq!(iter.next().unwrap(), &['s', 't']);
    assert!(iter.next().is_none());
    • \n
  • chunks(&self, chunk_size: usize) Returns an iterator over chunk_size elements of the slice at a time
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert_eq!(iter.next().unwrap(), &['m']);
    assert!(iter.next().is_none());
    • \n
  • chunks_mut()
    • \n
  • chunks_exact(&self, chunk_size: usize)
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks_exact(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert!(iter.next().is_none());
    assert_eq!(iter.remainder(), &['m']);
    • \n
  • as_chunks_unchecked<const N: usize>(&self) Splits the slice into a slice of N-element arrays, assuming that there’s no remainder
    • \n
  • as_chunks<const N: usize>(&self) Splits the slice into a slice of N-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than N
    • \n
  • as_rchunks<const N: usize>(&self) r means reverse
    • \n
  • group_by<F>(&self, pred: F) Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on slice[0] and slice[1] then on slice[1] and slice[2] and so on
    1
    2
    3
    4
    5
    6
    7
    #![feature(slice_group_by)]
    let slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];
    let mut iter = slice.group_by(|a, b| a <= b);
    assert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3, 4][..]));
    assert_eq!(iter.next(), None);
    • \n
  • split_at(&self, mid: usize) Divides one slice into two at an index.
    • \n
  • split<F>(&self, pred: F) Returns an iterator over subslices separated by elements that match pred. The matched element is not contained in the subslices.
    • \n
  • splitn<F>(&self, n: usize, pred: F)
    • \n
  • contains(&self, x: &T) Returns true if the slice contains an element with the given value.
    • \n
  • starts_with(&self, needle: &[T]) eturns true if needle is a prefix of the slice
    1
    2
    3
    4
    let v = [10, 40, 30];
    assert!(v.starts_with(&[10]));
    assert!(v.starts_with(&[10, 40]));
    assert!(!v.starts_with(&[50]));
    • \n
  • ends_with(&self, needle: &[T])
    • \n
  • strip_prefix Returns a subslice with the prefix removed.
    1
    2
    3
    4
    5
    6
    let v = &[10, 40, 30];
    assert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));
    assert_eq!(v.strip_prefix(&[50]), None);
    let prefix : &str = "he";
    assert_eq!(b"hello".strip_prefix(prefix.as_bytes()),
               Some(b"llo".as_ref()));
    • \n
  • strip_suffix
    • \n
  • binary_search(&self, x: &T) Binary searches this slice for a given element.
    • \n
  • sort_unstable(&mut self) Sorts the slice, but might not preserve the order of equal elements.
    • \n
  • rotate_left(&mut self, mid: usize) Rotates the slice in-place such that the first mid elements of the slice move to the end while the last self.len() - mid elements move to the front.
    • \n
  • fill(&mut self, value: T) Fills self with elements by cloning value.
    • \n
  • clone_from_slice(&mut self, src: &[T]) Copies the elements from src into self.
    • \n
  • copy_from_slice(&mut self, src: &[T])
    • \n
  • is_sorted(&self)
    • \n
  • take<'a, R: OneSidedRange<usize>>(self: &mut &'a Self, range: R) Removes the subslice corresponding to the given range
    • \n
  • get_many_mut<const N: usize> Returns mutable references to many indices at once.
    1
    2
    3
    4
    5
    6
    7
    #![feature(get_many_mut)]
    let v = &mut [1, 2, 3];
    if let Ok([a, b]) = v.get_many_mut([0, 2]) {
        *a = 413;
        *b = 612;
    }
    assert_eq!(v, &[413, 2, 612]);
    • \n
\n

alloc::vec::Vec

    \n
  • fn truncate(&mut self, len: usize) Shortens the vector, keeping the first len elements and dropping the rest
    • \n
\n

std::collections::VecDeque

A double-ended queue (deque) implemented with a growable ring buffer.
Since VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.

\n
    \n
  • swap(&mut self, i: usize, j: usize)
    • \n
  • reserve_exact(&mut self, additional: usize) Reserves the minimum capacity for at least additional more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.
    • \n
  • reserve(&mut self, additional: usize)
    • \n
  • shrink_to_fit(&mut self) Shrinks the capacity of the deque as much as possible.
    • \n
  • truncate(&mut self, len: usize) Shortens the deque, keeping the first len elements and dropping the rest.
    1
    2
    3
    4
    5
    6
    7
    8
    use std::collections::VecDeque;
    let mut buf = VecDeque::new();
    buf.push_back(5);
    buf.push_back(10);
    buf.push_back(15);
    assert_eq!(buf, [5, 10, 15]);
    buf.truncate(1);
    assert_eq!(buf, [5]);
    • \n
  • iter(&self)
    • \n
  • as_slices(&self)
    • \n
  • slice_ranges<R>(&self, range: R) Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range
    • \n
  • range<R>(&self, range: R) Creates an iterator that covers the specified range in the deque.
    1
    2
    3
    4
    5
    6
    7
    use std::collections::VecDeque;
    let deque: VecDeque<_> = [1, 2, 3].into();
    let range = deque.range(2..).copied().collect::<VecDeque<_>>();
    assert_eq!(range, [3]);
    // A full range covers all contents
    let all = deque.range(..);
    assert_eq!(all.len(), 3);
    • \n
  • drain<R>(&mut self, range: R) Removes the specified range from the deque in bulk, returning all removed elements as an iterator.
    • \n
  • clear(&mut self)
    • \n
  • contains(&self, x: &T) Returns true if the deque contains an element equal to the given value
    • \n
  • front(&self) Provides a reference to the front element
    • \n
  • front_mut(&mut self)
    • \n
  • back(&self)
    • \n
  • back_mut(&mut self)
    • \n
  • pop_front(&mut self)
    • \n
  • pop_back(&mut self)
    • \n
  • push_front(&mut self, value: T)
    • \n
  • push_back(&mut self, value: T)
    • \n
\n

std::collections::LinkedList

","site":{"data":{}},"excerpt":"","more":"

array

A fixed-size array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.

\n
1
todo!()
\n\n

slice

A dynamically-sized view into a contiguous sequence, [T].

\n
    \n
  • len(): Returns the number of elements in the slice
    • \n
  • is_empty()
    • \n
  • first() Returns the first element of the slice, or None if it is empty.
    • \n
  • first_mut() Returns a mutable pointer to the first element of the slice, or None if it is empty
    • \n
  • split_first() Returns the first and all the rest of the elements of the slice, or None if it is empty.
    • \n
  • split_first_mut()
    • \n
  • split_last()
    • \n
  • split_last_mut()
    • \n
  • last()
    • \n
  • last_mut()
    • \n
  • get<I>(index: I) Returns a reference to an element or subslice depending on the type of index.
    1
    2
    3
    let v = [10, 40, 30];
    assert_eq!(Some(&40), v.get(1));
    assert_eq!(Some(&[10, 40][..]), v.get(0..2));
    • \n
  • get_mut<I>(index: I)
    • \n
  • get_unchecked<I>(index: I) Returns a reference to an element or subslice, without doing bounds checking
    • \n
  • get_unchecked_mut<I>(index: I)
    • \n
  • as_ptr(&self) -> *const T Returns a raw pointer to the slice’s buffer
    1
    2
    3
    4
    5
    6
    7
    let x = &[1, 2, 4];
    let x_ptr = x.as_ptr();
    unsafe {
        for i in 0..x.len() {
            assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));
        }
    }
    • \n
  • as_mut_ptr(&mut self) -> *mut T 
    1
    2
    3
    4
    5
    6
    7
    8
    let x = &mut [1, 2, 4];
    let x_ptr = x.as_mut_ptr();
    unsafe {
        for i in 0..x.len() {
            *x_ptr.add(i) += 2;
        }
    }
    assert_eq!(x, &[3, 4, 6]);
    • \n
  • as_ptr_range(&self) -> Range<*const T> Returns the two raw pointers spanning the slice.
    1
    2
    3
    4
    5
    pub const fn as_ptr_range(&self) -> Range<*const T> {
        let start = self.as_ptr();
        let end = unsafe { start.add(self.len()) };
        start..end
    }
    • \n
  • as_mut_ptr_range(&mut self) -> Range<*mut T>
    • \n
  • swap(&mut self, a: usize, b: usize) Swaps two elements in the slice.
    • \n
  • reverse(&mut self) Reverses the order of elements in the slice, in place.
    • \n
  • windows(&self, size: usize) Returns an iterator over all contiguous windows of length size. The windows overlap. If the slice is shorter than size, the iterator returns no values.
    1
    2
    3
    4
    5
    6
    let slice = ['r', 'u', 's', 't'];
    let mut iter = slice.windows(2);
    assert_eq!(iter.next().unwrap(), &['r', 'u']);
    assert_eq!(iter.next().unwrap(), &['u', 's']);
    assert_eq!(iter.next().unwrap(), &['s', 't']);
    assert!(iter.next().is_none());
    • \n
  • chunks(&self, chunk_size: usize) Returns an iterator over chunk_size elements of the slice at a time
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert_eq!(iter.next().unwrap(), &['m']);
    assert!(iter.next().is_none());
    • \n
  • chunks_mut()
    • \n
  • chunks_exact(&self, chunk_size: usize)
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks_exact(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert!(iter.next().is_none());
    assert_eq!(iter.remainder(), &['m']);
    • \n
  • as_chunks_unchecked<const N: usize>(&self) Splits the slice into a slice of N-element arrays, assuming that there’s no remainder
    • \n
  • as_chunks<const N: usize>(&self) Splits the slice into a slice of N-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than N
    • \n
  • as_rchunks<const N: usize>(&self) r means reverse
    • \n
  • group_by<F>(&self, pred: F) Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on slice[0] and slice[1] then on slice[1] and slice[2] and so on
    1
    2
    3
    4
    5
    6
    7
    #![feature(slice_group_by)]
    let slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];
    let mut iter = slice.group_by(|a, b| a <= b);
    assert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3, 4][..]));
    assert_eq!(iter.next(), None);
    • \n
  • split_at(&self, mid: usize) Divides one slice into two at an index.
    • \n
  • split<F>(&self, pred: F) Returns an iterator over subslices separated by elements that match pred. The matched element is not contained in the subslices.
    • \n
  • splitn<F>(&self, n: usize, pred: F)
    • \n
  • contains(&self, x: &T) Returns true if the slice contains an element with the given value.
    • \n
  • starts_with(&self, needle: &[T]) eturns true if needle is a prefix of the slice
    1
    2
    3
    4
    let v = [10, 40, 30];
    assert!(v.starts_with(&[10]));
    assert!(v.starts_with(&[10, 40]));
    assert!(!v.starts_with(&[50]));
    • \n
  • ends_with(&self, needle: &[T])
    • \n
  • strip_prefix Returns a subslice with the prefix removed.
    1
    2
    3
    4
    5
    6
    let v = &[10, 40, 30];
    assert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));
    assert_eq!(v.strip_prefix(&[50]), None);
    let prefix : &str = "he";
    assert_eq!(b"hello".strip_prefix(prefix.as_bytes()),
               Some(b"llo".as_ref()));
    • \n
  • strip_suffix
    • \n
  • binary_search(&self, x: &T) Binary searches this slice for a given element.
    • \n
  • sort_unstable(&mut self) Sorts the slice, but might not preserve the order of equal elements.
    • \n
  • rotate_left(&mut self, mid: usize) Rotates the slice in-place such that the first mid elements of the slice move to the end while the last self.len() - mid elements move to the front.
    • \n
  • fill(&mut self, value: T) Fills self with elements by cloning value.
    • \n
  • clone_from_slice(&mut self, src: &[T]) Copies the elements from src into self.
    • \n
  • copy_from_slice(&mut self, src: &[T])
    • \n
  • is_sorted(&self)
    • \n
  • take<'a, R: OneSidedRange<usize>>(self: &mut &'a Self, range: R) Removes the subslice corresponding to the given range
    • \n
  • get_many_mut<const N: usize> Returns mutable references to many indices at once.
    1
    2
    3
    4
    5
    6
    7
    #![feature(get_many_mut)]
    let v = &mut [1, 2, 3];
    if let Ok([a, b]) = v.get_many_mut([0, 2]) {
        *a = 413;
        *b = 612;
    }
    assert_eq!(v, &[413, 2, 612]);
    • \n
\n

alloc::vec::Vec

    \n
  • fn truncate(&mut self, len: usize) Shortens the vector, keeping the first len elements and dropping the rest
    • \n
\n

std::collections::VecDeque

A double-ended queue (deque) implemented with a growable ring buffer.
Since VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.

\n
    \n
  • swap(&mut self, i: usize, j: usize)
    • \n
  • reserve_exact(&mut self, additional: usize) Reserves the minimum capacity for at least additional more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.
    • \n
  • reserve(&mut self, additional: usize)
    • \n
  • shrink_to_fit(&mut self) Shrinks the capacity of the deque as much as possible.
    • \n
  • truncate(&mut self, len: usize) Shortens the deque, keeping the first len elements and dropping the rest.
    1
    2
    3
    4
    5
    6
    7
    8
    use std::collections::VecDeque;
    let mut buf = VecDeque::new();
    buf.push_back(5);
    buf.push_back(10);
    buf.push_back(15);
    assert_eq!(buf, [5, 10, 15]);
    buf.truncate(1);
    assert_eq!(buf, [5]);
    • \n
  • iter(&self)
    • \n
  • as_slices(&self)
    • \n
  • slice_ranges<R>(&self, range: R) Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range
    • \n
  • range<R>(&self, range: R) Creates an iterator that covers the specified range in the deque.
    1
    2
    3
    4
    5
    6
    7
    use std::collections::VecDeque;
    let deque: VecDeque<_> = [1, 2, 3].into();
    let range = deque.range(2..).copied().collect::<VecDeque<_>>();
    assert_eq!(range, [3]);
    // A full range covers all contents
    let all = deque.range(..);
    assert_eq!(all.len(), 3);
    • \n
  • drain<R>(&mut self, range: R) Removes the specified range from the deque in bulk, returning all removed elements as an iterator.
    • \n
  • clear(&mut self)
    • \n
  • contains(&self, x: &T) Returns true if the deque contains an element equal to the given value
    • \n
  • front(&self) Provides a reference to the front element
    • \n
  • front_mut(&mut self)
    • \n
  • back(&self)
    • \n
  • back_mut(&mut self)
    • \n
  • pop_front(&mut self)
    • \n
  • pop_back(&mut self)
    • \n
  • push_front(&mut self, value: T)
    • \n
  • push_back(&mut self, value: T)
    • \n
\n

std::collections::LinkedList

"},{"title":"rust std data structure (2D)","date":"2023-05-02T14:04:38.000Z","_content":"\n## collections\n### BTreeMap\n- `clear(&mut self)` Clears the map, removing all elements.\n- `get(&self, key: &Q)` Returns a reference to the value corresponding to the key.\n- `get_key_value(&self, k: &Q)`\n- `first_key_value(&self)` eturns the first key-value pair in the map.\n- `first_entry(&mut self)` Returns the first entry in the map for in-place manipulation\n- `pop_first(&mut self)` \n- `last_key_value(&self)`\n- `last_entry(&mut self)`\n- `pop_last(&mut self)`\n- `contains_key(&self, key: &Q)`\n- `get_mut(&mut self, key: &Q)` Returns a mutable reference to the value corresponding to the key\n- `insert(&mut self, key: K, value: V)`\n- `try_insert(&mut self, key: K, value: V)` If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.\n- `remove(&mut self, key: &Q)`\n- `remove_entry(&mut self, key: &Q)`\n- `retain(&mut self, mut f: F)` Retains only the elements specified by the predicate.\n```rust\nuse std::collections::BTreeMap;\nlet mut map: BTreeMap = (0..8).map(|x| (x, x*10)).collect();\n// Keep only the elements with even-numbered keys.\nmap.retain(|&k, _| k % 2 == 0);\nassert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));\n```\n- `append(&mut self, other: &mut Self)` Moves all elements from `other` into `self`, leaving `other` empty.\n- `range(&self, range: R) -> Range<'_, K, V>` Constructs a double-ended iterator over a sub-range of elements in the map.\n```rust\nuse std::collections::BTreeMap;\nuse std::ops::Bound::Included;\nlet mut map = BTreeMap::new();\nmap.insert(3, \"a\");\nmap.insert(5, \"b\");\nmap.insert(8, \"c\");\nfor (&key, &value) in map.range((Included(&4), Included(&8))) {\n println!(\"{key}: {value}\");\n}\nassert_eq!(Some((&5, &\"b\")), map.range(4..).next());\n```\n- `range_mut(&mut self, range: R) -> RangeMut<'_, K, V>` \n- `entry(&mut self, key: K)` Gets the given key's corresponding entry in the map for in-place manipulation.\n```rust\nuse std::collections::BTreeMap;\nlet mut count: BTreeMap<&str, usize> = BTreeMap::new();\n// count the number of occurrences of letters in the vec\nfor x in [\"a\", \"b\", \"a\", \"c\", \"a\", \"b\"] {\n count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);\n}\nassert_eq!(count[\"a\"], 3);\nassert_eq!(count[\"b\"], 2);\nassert_eq!(count[\"c\"], 1);\n```\n- `split_off(&mut self, key: &Q)` Splits the collection into two at the given key. Returns everything after the given key,\n- `drain_filter(&mut self, pred: F)` Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns `true`, the element is removed from the map and yielded. If the closure returns `false`, or panics, the element remains in the map and will not be yielded\n- `into_keys(self)` Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this\n- `into_values(self)`\n","source":"_posts/rust/rust_std/rust-std-data-structure-2.md","raw":"---\ntitle: rust std data structure (2D)\ndate: 2023-05-02 22:04:38\ntags: [rust-std]\n---\n\n## collections\n### BTreeMap\n- `clear(&mut self)` Clears the map, removing all elements.\n- `get(&self, key: &Q)` Returns a reference to the value corresponding to the key.\n- `get_key_value(&self, k: &Q)`\n- `first_key_value(&self)` eturns the first key-value pair in the map.\n- `first_entry(&mut self)` Returns the first entry in the map for in-place manipulation\n- `pop_first(&mut self)` \n- `last_key_value(&self)`\n- `last_entry(&mut self)`\n- `pop_last(&mut self)`\n- `contains_key(&self, key: &Q)`\n- `get_mut(&mut self, key: &Q)` Returns a mutable reference to the value corresponding to the key\n- `insert(&mut self, key: K, value: V)`\n- `try_insert(&mut self, key: K, value: V)` If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.\n- `remove(&mut self, key: &Q)`\n- `remove_entry(&mut self, key: &Q)`\n- `retain(&mut self, mut f: F)` Retains only the elements specified by the predicate.\n```rust\nuse std::collections::BTreeMap;\nlet mut map: BTreeMap = (0..8).map(|x| (x, x*10)).collect();\n// Keep only the elements with even-numbered keys.\nmap.retain(|&k, _| k % 2 == 0);\nassert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));\n```\n- `append(&mut self, other: &mut Self)` Moves all elements from `other` into `self`, leaving `other` empty.\n- `range(&self, range: R) -> Range<'_, K, V>` Constructs a double-ended iterator over a sub-range of elements in the map.\n```rust\nuse std::collections::BTreeMap;\nuse std::ops::Bound::Included;\nlet mut map = BTreeMap::new();\nmap.insert(3, \"a\");\nmap.insert(5, \"b\");\nmap.insert(8, \"c\");\nfor (&key, &value) in map.range((Included(&4), Included(&8))) {\n println!(\"{key}: {value}\");\n}\nassert_eq!(Some((&5, &\"b\")), map.range(4..).next());\n```\n- `range_mut(&mut self, range: R) -> RangeMut<'_, K, V>` \n- `entry(&mut self, key: K)` Gets the given key's corresponding entry in the map for in-place manipulation.\n```rust\nuse std::collections::BTreeMap;\nlet mut count: BTreeMap<&str, usize> = BTreeMap::new();\n// count the number of occurrences of letters in the vec\nfor x in [\"a\", \"b\", \"a\", \"c\", \"a\", \"b\"] {\n count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);\n}\nassert_eq!(count[\"a\"], 3);\nassert_eq!(count[\"b\"], 2);\nassert_eq!(count[\"c\"], 1);\n```\n- `split_off(&mut self, key: &Q)` Splits the collection into two at the given key. Returns everything after the given key,\n- `drain_filter(&mut self, pred: F)` Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns `true`, the element is removed from the map and yielded. If the closure returns `false`, or panics, the element remains in the map and will not be yielded\n- `into_keys(self)` Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this\n- `into_values(self)`\n","slug":"rust/rust_std/rust-std-data-structure-2","published":1,"updated":"2023-07-05T13:41:15.719Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzw0029ofsjgx9d6544","content":"

collections

BTreeMap

    \n
  • clear(&mut self) Clears the map, removing all elements.
    • \n
  • get(&self, key: &Q) Returns a reference to the value corresponding to the key.
    • \n
  • get_key_value(&self, k: &Q)
    • \n
  • first_key_value(&self) eturns the first key-value pair in the map.
    • \n
  • first_entry(&mut self) Returns the first entry in the map for in-place manipulation
    • \n
  • pop_first(&mut self)
    • \n
  • last_key_value(&self)
    • \n
  • last_entry(&mut self)
    • \n
  • pop_last(&mut self)
    • \n
  • contains_key(&self, key: &Q)
    • \n
  • get_mut(&mut self, key: &Q) Returns a mutable reference to the value corresponding to the key
    • \n
  • insert(&mut self, key: K, value: V)
    • \n
  • try_insert(&mut self, key: K, value: V) If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.
    • \n
  • remove(&mut self, key: &Q)
    • \n
  • remove_entry(&mut self, key: &Q)
    • \n
  • retain<F>(&mut self, mut f: F) Retains only the elements specified by the predicate.
    1
    2
    3
    4
    5
    use std::collections::BTreeMap;
    let mut map: BTreeMap<i32, i32> = (0..8).map(|x| (x, x*10)).collect();
    // Keep only the elements with even-numbered keys.
    map.retain(|&k, _| k % 2 == 0);
    assert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));
    • \n
  • append(&mut self, other: &mut Self) Moves all elements from other into self, leaving other empty.
    • \n
  • range<T: ?Sized, R>(&self, range: R) -> Range<'_, K, V> Constructs a double-ended iterator over a sub-range of elements in the map.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::collections::BTreeMap;
    use std::ops::Bound::Included;
    let mut map = BTreeMap::new();
    map.insert(3, "a");
    map.insert(5, "b");
    map.insert(8, "c");
    for (&key, &value) in map.range((Included(&4), Included(&8))) {
        println!("{key}: {value}");
    }
    assert_eq!(Some((&5, &"b")), map.range(4..).next());
    • \n
  • range_mut<T: ?Sized, R>(&mut self, range: R) -> RangeMut<'_, K, V>
    • \n
  • entry(&mut self, key: K) Gets the given key’s corresponding entry in the map for in-place manipulation.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    use std::collections::BTreeMap;
    let mut count: BTreeMap<&str, usize> = BTreeMap::new();
    // count the number of occurrences of letters in the vec
    for x in ["a", "b", "a", "c", "a", "b"] {
        count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);
    }
    assert_eq!(count["a"], 3);
    assert_eq!(count["b"], 2);
    assert_eq!(count["c"], 1);
    • \n
  • split_off<Q: ?Sized + Ord>(&mut self, key: &Q) Splits the collection into two at the given key. Returns everything after the given key,
    • \n
  • drain_filter<F>(&mut self, pred: F) Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns true, the element is removed from the map and yielded. If the closure returns false, or panics, the element remains in the map and will not be yielded
    • \n
  • into_keys(self) Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this
    • \n
  • into_values(self)
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

collections

BTreeMap

    \n
  • clear(&mut self) Clears the map, removing all elements.
    • \n
  • get(&self, key: &Q) Returns a reference to the value corresponding to the key.
    • \n
  • get_key_value(&self, k: &Q)
    • \n
  • first_key_value(&self) eturns the first key-value pair in the map.
    • \n
  • first_entry(&mut self) Returns the first entry in the map for in-place manipulation
    • \n
  • pop_first(&mut self)
    • \n
  • last_key_value(&self)
    • \n
  • last_entry(&mut self)
    • \n
  • pop_last(&mut self)
    • \n
  • contains_key(&self, key: &Q)
    • \n
  • get_mut(&mut self, key: &Q) Returns a mutable reference to the value corresponding to the key
    • \n
  • insert(&mut self, key: K, value: V)
    • \n
  • try_insert(&mut self, key: K, value: V) If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.
    • \n
  • remove(&mut self, key: &Q)
    • \n
  • remove_entry(&mut self, key: &Q)
    • \n
  • retain<F>(&mut self, mut f: F) Retains only the elements specified by the predicate.
    1
    2
    3
    4
    5
    use std::collections::BTreeMap;
    let mut map: BTreeMap<i32, i32> = (0..8).map(|x| (x, x*10)).collect();
    // Keep only the elements with even-numbered keys.
    map.retain(|&k, _| k % 2 == 0);
    assert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));
    • \n
  • append(&mut self, other: &mut Self) Moves all elements from other into self, leaving other empty.
    • \n
  • range<T: ?Sized, R>(&self, range: R) -> Range<'_, K, V> Constructs a double-ended iterator over a sub-range of elements in the map.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::collections::BTreeMap;
    use std::ops::Bound::Included;
    let mut map = BTreeMap::new();
    map.insert(3, "a");
    map.insert(5, "b");
    map.insert(8, "c");
    for (&key, &value) in map.range((Included(&4), Included(&8))) {
        println!("{key}: {value}");
    }
    assert_eq!(Some((&5, &"b")), map.range(4..).next());
    • \n
  • range_mut<T: ?Sized, R>(&mut self, range: R) -> RangeMut<'_, K, V>
    • \n
  • entry(&mut self, key: K) Gets the given key’s corresponding entry in the map for in-place manipulation.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    use std::collections::BTreeMap;
    let mut count: BTreeMap<&str, usize> = BTreeMap::new();
    // count the number of occurrences of letters in the vec
    for x in ["a", "b", "a", "c", "a", "b"] {
        count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);
    }
    assert_eq!(count["a"], 3);
    assert_eq!(count["b"], 2);
    assert_eq!(count["c"], 1);
    • \n
  • split_off<Q: ?Sized + Ord>(&mut self, key: &Q) Splits the collection into two at the given key. Returns everything after the given key,
    • \n
  • drain_filter<F>(&mut self, pred: F) Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns true, the element is removed from the map and yielded. If the closure returns false, or panics, the element remains in the map and will not be yielded
    • \n
  • into_keys(self) Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this
    • \n
  • into_values(self)
    • \n
\n"},{"title":"rust std sync","date":"2023-06-08T14:04:38.000Z","_content":"\n## [lock free & wait free](https://en.wikipedia.org/wiki/Non-blocking_algorithm)\n\"Lock-free\" and \"wait-free\" are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.\n- **lock-free** A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.\n- **wait-free** A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.\n\nIt's important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.\n\n## [atomic](https://doc.rust-lang.org/stable/std/sync/atomic/index.html)\nRust atomics currently follow the same rules as [C++20 atomics](https://en.cppreference.com/w/cpp/atomic), specifically `atomic_ref`. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an `atomic_ref` in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends. \nEach method takes an `Ordering` which represents the strength of the memory barrier for that operation. These orderings are the same as the [C++20 atomic orderings](https://en.cppreference.com/w/cpp/atomic/memory_order). For more information see the [nomicon](https://doc.rust-lang.org/stable/nomicon/atomics.html)\nAtomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).\n\n### Compiler Reordering\nCompilers may change the actual order of events, or make events never occur! If we write something like\n```rust\nx = 1;\ny = 3;\nx = 2;\n```\nThe compiler may conclude that it would be best if your program did:\n```rust\nx = 2;\ny = 3;\n```\nThis has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned. \n\n### Hardware Reordering\nhere is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn't actually have that memory in cache. The end result is that the hardware doesn't guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.\nFor instance, say we convince the compiler to emit this logic:\n```\ninitial state: x = 0, y = 1\n\nTHREAD 1 THREAD2\ny = 3; if x == 1 {\nx = 1; y *= 2;\n }\n```\nIdeally this program has 2 possible final states:\n- y = 3: (thread 2 did the check before thread 1 completed)\n- y = 6: (thread 2 did the check after thread 1 completed)\nHowever there's a third potential state that the hardware enables:\n- y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)\nIt's worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees. \n\n### Data Accesses\nAtomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:\n- Sequentially Consistent (SeqCst)\n- Release\n- Acquire\n- Relaxed\n\n### Sequentially Consistent\nSequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.\n\n### Acquire-Release\nAcquire and Release are largely intended to be paired. they're perfectly suited for acquiring and releasing locks. \nIntuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::thread;\n\nfn main() {\n let lock = Arc::new(AtomicBool::new(false)); // value answers \"am I locked?\"\n\n // ... distribute lock to threads somehow ...\n\n // Try to acquire the lock by setting it to true\n while lock.compare_and_swap(false, true, Ordering::Acquire) { }\n // broke out of the loop, so we successfully acquired the lock!\n\n // ... scary data accesses ...\n\n // ok we're done, release the lock\n lock.store(false, Ordering::Release);\n}\n```\n### Relaxed\nRelaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don't count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed `fetch_add` if you're not using the counter to synchronize any other accesses.\n\n## an example (spinlock)\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::{hint, thread};\n\nfn main() {\n let spinlock = Arc::new(AtomicUsize::new(1));\n\n let spinlock_clone = Arc::clone(&spinlock);\n let thread = thread::spawn(move|| {\n spinlock_clone.store(0, Ordering::SeqCst);\n });\n\n // Wait for the other thread to release the lock\n while spinlock.load(Ordering::SeqCst) != 0 {\n hint::spin_loop();\n }\n\n if let Err(panic) = thread.join() {\n println!(\"Thread had an error: {panic:?}\");\n }\n}\n```\n\n## usual structs\n1. [AtomicBool](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicBool.html)\n### methods\n- `fn get_mut(&mut self) -> &mut bool`\n- `fn into_inner(self) -> bool`\n- `fn load(&self, order: Ordering) -> bool`\n- `fn store(&self, val: bool, order: Ordering)`\n- `fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result`\nStores a value into the bool if the current value is the same as the current value.\ncompare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails. \n- `fn fetch_and(&self, val: bool, order: Ordering) -> bool`\nLogical “and” with a boolean value.\nPerforms a logical “and” operation on the current value and the argument val, and sets the new value to the result.\n- `const fn as_ptr(&self) -> *mut bool`\nReturns a mutable pointer to the underlying bool.\nDoing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.\n\n2. [AtomicUsize](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicUsize.html)\n2. [AtomicPtr](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html)\nA raw pointer type which can be safely shared between threads.\nThis type has the same in-memory representation as a *mut T\n\n\n## Higher-level synchronization objects\nMost of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.\n- **Arc**: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.\n- **Barrier**: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.\n- **Condvar**: Condition Variable, providing the ability to block a thread while waiting for an event to occur.\n- **mpsc**: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.\n- **Mutex**: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.\n- **Once**: Used for a thread-safe, one-time global initialization routine\n- **OnceLock**: Used for thread-safe, one-time initialization of a global variable.\n- **RwLock**: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.\n\n## mpsc\nThis module provides message-based communication over channels, concretely defined among three types:\n- Sender\n- SyncSender\n- Receiver","source":"_posts/rust/rust_std/rust-std-sync.md","raw":"---\ntitle: rust std sync\ndate: 2023-06-08 22:04:38\ntags: [rust-std]\n---\n\n## [lock free & wait free](https://en.wikipedia.org/wiki/Non-blocking_algorithm)\n\"Lock-free\" and \"wait-free\" are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.\n- **lock-free** A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.\n- **wait-free** A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.\n\nIt's important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.\n\n## [atomic](https://doc.rust-lang.org/stable/std/sync/atomic/index.html)\nRust atomics currently follow the same rules as [C++20 atomics](https://en.cppreference.com/w/cpp/atomic), specifically `atomic_ref`. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an `atomic_ref` in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends. \nEach method takes an `Ordering` which represents the strength of the memory barrier for that operation. These orderings are the same as the [C++20 atomic orderings](https://en.cppreference.com/w/cpp/atomic/memory_order). For more information see the [nomicon](https://doc.rust-lang.org/stable/nomicon/atomics.html)\nAtomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).\n\n### Compiler Reordering\nCompilers may change the actual order of events, or make events never occur! If we write something like\n```rust\nx = 1;\ny = 3;\nx = 2;\n```\nThe compiler may conclude that it would be best if your program did:\n```rust\nx = 2;\ny = 3;\n```\nThis has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned. \n\n### Hardware Reordering\nhere is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn't actually have that memory in cache. The end result is that the hardware doesn't guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.\nFor instance, say we convince the compiler to emit this logic:\n```\ninitial state: x = 0, y = 1\n\nTHREAD 1 THREAD2\ny = 3; if x == 1 {\nx = 1; y *= 2;\n }\n```\nIdeally this program has 2 possible final states:\n- y = 3: (thread 2 did the check before thread 1 completed)\n- y = 6: (thread 2 did the check after thread 1 completed)\nHowever there's a third potential state that the hardware enables:\n- y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)\nIt's worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees. \n\n### Data Accesses\nAtomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:\n- Sequentially Consistent (SeqCst)\n- Release\n- Acquire\n- Relaxed\n\n### Sequentially Consistent\nSequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.\n\n### Acquire-Release\nAcquire and Release are largely intended to be paired. they're perfectly suited for acquiring and releasing locks. \nIntuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::thread;\n\nfn main() {\n let lock = Arc::new(AtomicBool::new(false)); // value answers \"am I locked?\"\n\n // ... distribute lock to threads somehow ...\n\n // Try to acquire the lock by setting it to true\n while lock.compare_and_swap(false, true, Ordering::Acquire) { }\n // broke out of the loop, so we successfully acquired the lock!\n\n // ... scary data accesses ...\n\n // ok we're done, release the lock\n lock.store(false, Ordering::Release);\n}\n```\n### Relaxed\nRelaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don't count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed `fetch_add` if you're not using the counter to synchronize any other accesses.\n\n## an example (spinlock)\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::{hint, thread};\n\nfn main() {\n let spinlock = Arc::new(AtomicUsize::new(1));\n\n let spinlock_clone = Arc::clone(&spinlock);\n let thread = thread::spawn(move|| {\n spinlock_clone.store(0, Ordering::SeqCst);\n });\n\n // Wait for the other thread to release the lock\n while spinlock.load(Ordering::SeqCst) != 0 {\n hint::spin_loop();\n }\n\n if let Err(panic) = thread.join() {\n println!(\"Thread had an error: {panic:?}\");\n }\n}\n```\n\n## usual structs\n1. [AtomicBool](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicBool.html)\n### methods\n- `fn get_mut(&mut self) -> &mut bool`\n- `fn into_inner(self) -> bool`\n- `fn load(&self, order: Ordering) -> bool`\n- `fn store(&self, val: bool, order: Ordering)`\n- `fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result`\nStores a value into the bool if the current value is the same as the current value.\ncompare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails. \n- `fn fetch_and(&self, val: bool, order: Ordering) -> bool`\nLogical “and” with a boolean value.\nPerforms a logical “and” operation on the current value and the argument val, and sets the new value to the result.\n- `const fn as_ptr(&self) -> *mut bool`\nReturns a mutable pointer to the underlying bool.\nDoing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.\n\n2. [AtomicUsize](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicUsize.html)\n2. [AtomicPtr](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html)\nA raw pointer type which can be safely shared between threads.\nThis type has the same in-memory representation as a *mut T\n\n\n## Higher-level synchronization objects\nMost of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.\n- **Arc**: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.\n- **Barrier**: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.\n- **Condvar**: Condition Variable, providing the ability to block a thread while waiting for an event to occur.\n- **mpsc**: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.\n- **Mutex**: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.\n- **Once**: Used for a thread-safe, one-time global initialization routine\n- **OnceLock**: Used for thread-safe, one-time initialization of a global variable.\n- **RwLock**: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.\n\n## mpsc\nThis module provides message-based communication over channels, concretely defined among three types:\n- Sender\n- SyncSender\n- Receiver","slug":"rust/rust_std/rust-std-sync","published":1,"updated":"2023-07-05T14:21:06.619Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clk4sjzzz0039ofsj72gx9dsm","content":"

lock free & wait free

“Lock-free” and “wait-free” are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.

\n
    \n
  • lock-free A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.
    • \n
  • wait-free A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.
    • \n
\n

It’s important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.

\n

atomic

Rust atomics currently follow the same rules as C++20 atomics, specifically atomic_ref. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an atomic_ref in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends.
Each method takes an Ordering which represents the strength of the memory barrier for that operation. These orderings are the same as the C++20 atomic orderings. For more information see the nomicon
Atomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).

\n

Compiler Reordering

Compilers may change the actual order of events, or make events never occur! If we write something like

\n
1
2
3
x = 1;
y = 3;
x = 2;
\n

The compiler may conclude that it would be best if your program did:

\n
1
2
x = 2;
y = 3;
\n

This has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned.

\n

Hardware Reordering

here is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn’t actually have that memory in cache. The end result is that the hardware doesn’t guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.
For instance, say we convince the compiler to emit this logic:

\n
1
2
3
4
5
6
initial state: x = 0, y = 1

THREAD 1        THREAD2
y = 3;          if x == 1 {
x = 1;              y *= 2;
                }
\n

Ideally this program has 2 possible final states:

\n
    \n
  • y = 3: (thread 2 did the check before thread 1 completed)
    • \n
  • y = 6: (thread 2 did the check after thread 1 completed)
    However there’s a third potential state that the hardware enables:
    • \n
  • y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)
    It’s worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees.
    • \n
\n

Data Accesses

Atomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:

\n
    \n
  • Sequentially Consistent (SeqCst)
    • \n
  • Release
    • \n
  • Acquire
    • \n
  • Relaxed
    • \n
\n

Sequentially Consistent

Sequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.

\n

Acquire-Release

Acquire and Release are largely intended to be paired. they’re perfectly suited for acquiring and releasing locks.
Intuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;

fn main() {
    let lock = Arc::new(AtomicBool::new(false)); // value answers "am I locked?"

    // ... distribute lock to threads somehow ...

    // Try to acquire the lock by setting it to true
    while lock.compare_and_swap(false, true, Ordering::Acquire) { }
    // broke out of the loop, so we successfully acquired the lock!

    // ... scary data accesses ...

    // ok we're done, release the lock
    lock.store(false, Ordering::Release);
}
\n

Relaxed

Relaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don’t count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed fetch_add if you’re not using the counter to synchronize any other accesses.

\n

an example (spinlock)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{hint, thread};

fn main() {
    let spinlock = Arc::new(AtomicUsize::new(1));

    let spinlock_clone = Arc::clone(&spinlock);
    let thread = thread::spawn(move|| {
        spinlock_clone.store(0, Ordering::SeqCst);
    });

    // Wait for the other thread to release the lock
    while spinlock.load(Ordering::SeqCst) != 0 {
        hint::spin_loop();
    }

    if let Err(panic) = thread.join() {
        println!("Thread had an error: {panic:?}");
    }
}
\n\n

usual structs

    \n
  1. AtomicBool
    2. \n
\n

methods

    \n
  • fn get_mut(&mut self) -> &mut bool
    • \n
  • fn into_inner(self) -> bool
    • \n
  • fn load(&self, order: Ordering) -> bool
    • \n
  • fn store(&self, val: bool, order: Ordering)
    • \n
  • fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result<bool, bool>
    Stores a value into the bool if the current value is the same as the current value.
    compare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails.
    • \n
  • fn fetch_and(&self, val: bool, order: Ordering) -> bool
    Logical “and” with a boolean value.
    Performs a logical “and” operation on the current value and the argument val, and sets the new value to the result.
    • \n
  • const fn as_ptr(&self) -> *mut bool
    Returns a mutable pointer to the underlying bool.
    Doing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.
    • \n
\n
    \n
  1. AtomicUsize
    2. \n
  2. AtomicPtr
    A raw pointer type which can be safely shared between threads.
    This type has the same in-memory representation as a *mut T
    3. \n
\n

Higher-level synchronization objects

Most of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.

\n
    \n
  • Arc: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.
    • \n
  • Barrier: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.
    • \n
  • Condvar: Condition Variable, providing the ability to block a thread while waiting for an event to occur.
    • \n
  • mpsc: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.
    • \n
  • Mutex: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.
    • \n
  • Once: Used for a thread-safe, one-time global initialization routine
    • \n
  • OnceLock: Used for thread-safe, one-time initialization of a global variable.
    • \n
  • RwLock: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.
    • \n
\n

mpsc

This module provides message-based communication over channels, concretely defined among three types:

\n
    \n
  • Sender
    • \n
  • SyncSender
    • \n
  • Receiver
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

lock free & wait free

“Lock-free” and “wait-free” are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.

\n
    \n
  • lock-free A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.
    • \n
  • wait-free A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.
    • \n
\n

It’s important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.

\n

atomic

Rust atomics currently follow the same rules as C++20 atomics, specifically atomic_ref. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an atomic_ref in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends.
Each method takes an Ordering which represents the strength of the memory barrier for that operation. These orderings are the same as the C++20 atomic orderings. For more information see the nomicon
Atomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).

\n

Compiler Reordering

Compilers may change the actual order of events, or make events never occur! If we write something like

\n
1
2
3
x = 1;
y = 3;
x = 2;
\n

The compiler may conclude that it would be best if your program did:

\n
1
2
x = 2;
y = 3;
\n

This has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned.

\n

Hardware Reordering

here is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn’t actually have that memory in cache. The end result is that the hardware doesn’t guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.
For instance, say we convince the compiler to emit this logic:

\n
1
2
3
4
5
6
initial state: x = 0, y = 1

THREAD 1        THREAD2
y = 3;          if x == 1 {
x = 1;              y *= 2;
                }
\n

Ideally this program has 2 possible final states:

\n
    \n
  • y = 3: (thread 2 did the check before thread 1 completed)
    • \n
  • y = 6: (thread 2 did the check after thread 1 completed)
    However there’s a third potential state that the hardware enables:
    • \n
  • y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)
    It’s worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees.
    • \n
\n

Data Accesses

Atomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:

\n
    \n
  • Sequentially Consistent (SeqCst)
    • \n
  • Release
    • \n
  • Acquire
    • \n
  • Relaxed
    • \n
\n

Sequentially Consistent

Sequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.

\n

Acquire-Release

Acquire and Release are largely intended to be paired. they’re perfectly suited for acquiring and releasing locks.
Intuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;

fn main() {
    let lock = Arc::new(AtomicBool::new(false)); // value answers "am I locked?"

    // ... distribute lock to threads somehow ...

    // Try to acquire the lock by setting it to true
    while lock.compare_and_swap(false, true, Ordering::Acquire) { }
    // broke out of the loop, so we successfully acquired the lock!

    // ... scary data accesses ...

    // ok we're done, release the lock
    lock.store(false, Ordering::Release);
}
\n

Relaxed

Relaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don’t count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed fetch_add if you’re not using the counter to synchronize any other accesses.

\n

an example (spinlock)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{hint, thread};

fn main() {
    let spinlock = Arc::new(AtomicUsize::new(1));

    let spinlock_clone = Arc::clone(&spinlock);
    let thread = thread::spawn(move|| {
        spinlock_clone.store(0, Ordering::SeqCst);
    });

    // Wait for the other thread to release the lock
    while spinlock.load(Ordering::SeqCst) != 0 {
        hint::spin_loop();
    }

    if let Err(panic) = thread.join() {
        println!("Thread had an error: {panic:?}");
    }
}
\n\n

usual structs

    \n
  1. AtomicBool
    2. \n
\n

methods

    \n
  • fn get_mut(&mut self) -> &mut bool
    • \n
  • fn into_inner(self) -> bool
    • \n
  • fn load(&self, order: Ordering) -> bool
    • \n
  • fn store(&self, val: bool, order: Ordering)
    • \n
  • fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result<bool, bool>
    Stores a value into the bool if the current value is the same as the current value.
    compare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails.
    • \n
  • fn fetch_and(&self, val: bool, order: Ordering) -> bool
    Logical “and” with a boolean value.
    Performs a logical “and” operation on the current value and the argument val, and sets the new value to the result.
    • \n
  • const fn as_ptr(&self) -> *mut bool
    Returns a mutable pointer to the underlying bool.
    Doing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.
    • \n
\n
    \n
  1. AtomicUsize
    2. \n
  2. AtomicPtr
    A raw pointer type which can be safely shared between threads.
    This type has the same in-memory representation as a *mut T
    3. \n
\n

Higher-level synchronization objects

Most of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.

\n
    \n
  • Arc: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.
    • \n
  • Barrier: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.
    • \n
  • Condvar: Condition Variable, providing the ability to block a thread while waiting for an event to occur.
    • \n
  • mpsc: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.
    • \n
  • Mutex: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.
    • \n
  • Once: Used for a thread-safe, one-time global initialization routine
    • \n
  • OnceLock: Used for thread-safe, one-time initialization of a global variable.
    • \n
  • RwLock: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.
    • \n
\n

mpsc

This module provides message-based communication over channels, concretely defined among three types:

\n
    \n
  • Sender
    • \n
  • SyncSender
    • \n
  • Receiver
    • \n
\n"}],"PostAsset":[],"PostCategory":[],"PostTag":[{"post_id":"clk4sjzze0000ofsjat5l27fl","tag_id":"clk4sjzzi0002ofsj5zk107n3","_id":"clk4sjzzm000bofsjeh2sf4k5"},{"post_id":"clk4sjzze0000ofsjat5l27fl","tag_id":"clk4sjzzk0006ofsjckwaff5b","_id":"clk4sjzzn000dofsjfyt56f24"},{"post_id":"clk4sjzzh0001ofsj9gqng05l","tag_id":"clk4sjzzi0002ofsj5zk107n3","_id":"clk4sjzzn000gofsj2gxs19ao"},{"post_id":"clk4sjzzn000fofsj4keu9ail","tag_id":"clk4sjzzn000eofsjebxy3e35","_id":"clk4sjzzn000jofsj19s55ozm"},{"post_id":"clk4sjzzk0005ofsj6z7xbxfl","tag_id":"clk4sjzzn000eofsjebxy3e35","_id":"clk4sjzzo000lofsj609ahpyg"},{"post_id":"clk4sjzzl0007ofsj05l8frvy","tag_id":"clk4sjzzn000eofsjebxy3e35","_id":"clk4sjzzo000pofsje97pex8z"},{"post_id":"clk4sjzzl0008ofsjc9z43usj","tag_id":"clk4sjzzn000eofsjebxy3e35","_id":"clk4sjzzp000tofsjajfx1k5z"},{"post_id":"clk4sjzzm000aofsj62am2966","tag_id":"clk4sjzzn000eofsjebxy3e35","_id":"clk4sjzzq000xofsja3ax77ik"},{"post_id":"clk4sjzzm000cofsj603lgdqw","tag_id":"clk4sjzzp000vofsjhlzv3ciq","_id":"clk4sjzzq0011ofsj9e2i8u27"},{"post_id":"clk4sjzzn000hofsj1c627es0","tag_id":"clk4sjzzn000eofsjebxy3e35","_id":"clk4sjzzr0019ofsjfexs2st4"},{"post_id":"clk4sjzzn000hofsj1c627es0","tag_id":"clk4sjzzq000zofsjeay7elog","_id":"clk4sjzzs001bofsjfnikha1c"},{"post_id":"clk4sjzzn000hofsj1c627es0","tag_id":"clk4sjzzr0014ofsjguy8bq8e","_id":"clk4sjzzs001eofsjf9ro7hcz"},{"post_id":"clk4sjzzo000kofsj3zwhcohs","tag_id":"clk4sjzzp000vofsjhlzv3ciq","_id":"clk4sjzzs001gofsj09t3fjic"},{"post_id":"clk4sjzzs001dofsj3nu0e5po","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzt001jofsjb1vpctty"},{"post_id":"clk4sjzzo000mofsj1fzd99u4","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzt001lofsjhohm8113"},{"post_id":"clk4sjzzs001fofsj1g9v6111","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzt001oofsj2t6sd666"},{"post_id":"clk4sjzzo000oofsj3jfs99sr","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzt001qofsj3tnfev2b"},{"post_id":"clk4sjzzt001kofsjh3ln39ny","tag_id":"clk4sjzzi0002ofsj5zk107n3","_id":"clk4sjzzu001tofsj2hpmc8eq"},{"post_id":"clk4sjzzt001kofsjh3ln39ny","tag_id":"clk4sjzzk0006ofsjckwaff5b","_id":"clk4sjzzu001vofsjgoq44dqz"},{"post_id":"clk4sjzzt001nofsj44p54fok","tag_id":"clk4sjzzi0002ofsj5zk107n3","_id":"clk4sjzzu001yofsja2j0hzic"},{"post_id":"clk4sjzzt001nofsj44p54fok","tag_id":"clk4sjzzk0006ofsjckwaff5b","_id":"clk4sjzzu0020ofsj7a0u6j9o"},{"post_id":"clk4sjzzo000qofsjg48t6z8b","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzv0023ofsjg7vr2smy"},{"post_id":"clk4sjzzt001pofsj2eunaq89","tag_id":"clk4sjzzi0002ofsj5zk107n3","_id":"clk4sjzzw0025ofsjdt8vgy0i"},{"post_id":"clk4sjzzt001pofsj2eunaq89","tag_id":"clk4sjzzk0006ofsjckwaff5b","_id":"clk4sjzzw0028ofsjd22bcmcv"},{"post_id":"clk4sjzzu001sofsjc0w8dddm","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzw002aofsj321n2m4b"},{"post_id":"clk4sjzzp000sofsj8j5g4s8r","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzx002cofsjd3x0fa4f"},{"post_id":"clk4sjzzu001xofsj4ipzhfxp","tag_id":"clk4sjzzk0006ofsjckwaff5b","_id":"clk4sjzzx002dofsjefowhobl"},{"post_id":"clk4sjzzp000uofsjgftu030k","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzx002fofsj9pgo6b9k"},{"post_id":"clk4sjzzu001zofsj6bkj0wad","tag_id":"clk4sjzzk0006ofsjckwaff5b","_id":"clk4sjzzx002gofsj4ezxd3d0"},{"post_id":"clk4sjzzv0022ofsjde824or5","tag_id":"clk4sjzzk0006ofsjckwaff5b","_id":"clk4sjzzx002iofsjfoyt5ziq"},{"post_id":"clk4sjzzq000wofsj7u5g7rf7","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzx002jofsjfpdj3fhn"},{"post_id":"clk4sjzzq000yofsjaa7u5z4f","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzx002lofsj0hjn23dm"},{"post_id":"clk4sjzzq0010ofsjfj0gc4g6","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzx002mofsj04900g45"},{"post_id":"clk4sjzzq0012ofsjcu8w4p8b","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzx002oofsj1ixxfbir"},{"post_id":"clk4sjzzr0013ofsj6wi71v7f","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzx002pofsjem5t8odi"},{"post_id":"clk4sjzzr0015ofsjfviv9rjn","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzx002rofsj63232sy9"},{"post_id":"clk4sjzzr0016ofsjdqxn9hhz","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzx002sofsj2xb25g0u"},{"post_id":"clk4sjzzr0018ofsj3y1gc8vv","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzy002vofsjas5gdzo7"},{"post_id":"clk4sjzzr0018ofsj3y1gc8vv","tag_id":"clk4sjzzx002tofsj76v4ga2x","_id":"clk4sjzzy002wofsja90hgo25"},{"post_id":"clk4sjzzr001aofsj3n9p8vvr","tag_id":"clk4sjzzs001cofsje0l51bxy","_id":"clk4sjzzy002yofsjdxu3b4sc"},{"post_id":"clk4sjzzt001iofsj5f219qa6","tag_id":"clk4sjzzn000eofsjebxy3e35","_id":"clk4sjzzy0030ofsjdmecg61q"},{"post_id":"clk4sjzzt001iofsj5f219qa6","tag_id":"clk4sjzzy002xofsj4xgrdl9p","_id":"clk4sjzzy0031ofsjb88a94fu"},{"post_id":"clk4sjzzu001uofsjhhpibdh9","tag_id":"clk4sjzzy002zofsj3n18dvmu","_id":"clk4sjzzy0033ofsj5xd22kew"},{"post_id":"clk4sjzzv0024ofsjcdqa4dbl","tag_id":"clk4sjzzy0032ofsj6rbo0ek1","_id":"clk4sjzzy0035ofsj3yis8ag1"},{"post_id":"clk4sjzzw0027ofsj952sej9f","tag_id":"clk4sjzzy0032ofsj6rbo0ek1","_id":"clk4sjzzy0037ofsjebplalte"},{"post_id":"clk4sjzzw0029ofsjgx9d6544","tag_id":"clk4sjzzy0032ofsj6rbo0ek1","_id":"clk4sjzzy0038ofsjcsfn2s9d"},{"post_id":"clk4sjzzz0039ofsj72gx9dsm","tag_id":"clk4sjzzy0032ofsj6rbo0ek1","_id":"clk4sjzzz003aofsjafo92pmk"}],"Tag":[{"name":"blockchain","_id":"clk4sjzzi0002ofsj5zk107n3"},{"name":"geth","_id":"clk4sjzzk0006ofsjckwaff5b"},{"name":"cryptography","_id":"clk4sjzzn000eofsjebxy3e35"},{"name":"golang","_id":"clk4sjzzp000vofsjhlzv3ciq"},{"name":"mpc","_id":"clk4sjzzq000zofsjeay7elog"},{"name":"ecdsa","_id":"clk4sjzzr0014ofsjguy8bq8e"},{"name":"rust","_id":"clk4sjzzs001cofsje0l51bxy"},{"name":"cargo","_id":"clk4sjzzx002tofsj76v4ga2x"},{"name":"zkp","_id":"clk4sjzzy002xofsj4xgrdl9p"},{"name":"rust-crate","_id":"clk4sjzzy002zofsj3n18dvmu"},{"name":"rust-std","_id":"clk4sjzzy0032ofsj6rbo0ek1"}]}} \ No newline at end of file diff --git a/public/2022/09/27/rust/rust-01-resource/index.html b/public/2022/09/27/rust/rust-01-resource/index.html index f5571508..ff7c2611 100644 --- a/public/2022/09/27/rust/rust-01-resource/index.html +++ b/public/2022/09/27/rust/rust-01-resource/index.html @@ -121,7 +121,7 @@

books

<