diff --git a/go.mod b/go.mod index b42ddc1..a4d255a 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,10 @@ require ( github.com/ipfs/go-cid v0.0.6 github.com/ipfs/go-ipld-cbor v0.0.4 github.com/ipfs/go-ipld-format v0.0.2 // indirect - github.com/kr/pretty v0.1.0 // indirect + github.com/ipld/go-ipld-prime v0.12.3 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.6.1 github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488 - golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index 150f9f3..ef54d1f 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= @@ -11,6 +15,7 @@ github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJ github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.6 h1:go0y+GcDOGeJIV01FeBsta4FHngoA4Wz7KMeLkXAhMs= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= @@ -20,22 +25,29 @@ github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9 github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= -github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= +github.com/ipld/go-ipld-prime v0.12.3 h1:furVobw7UBLQZwlEwfE26tYORy3PAK8VYSgZOSr3JMQ= +github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= @@ -43,41 +55,54 @@ github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoR github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multicodec v0.3.0 h1:tstDwfIjiHbnIjeM5Lp+pMrSeN+LCMsEwOrkPmWm03A= +github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM= +github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= +github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 h1:bzMe+2coZJYHnhGgVlcQKuRy4FSny4ds8dLQjw5P1XE= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= +github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc= +github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488 h1:P/Q9QT99FpyHtFke7ERUqX7yYtZ/KigO880L+TKFyTQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/hamt.go b/hamt.go index e57b8c2..6465ae5 100644 --- a/hamt.go +++ b/hamt.go @@ -9,6 +9,7 @@ import ( cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" + "github.com/ipld/go-ipld-prime" cbg "github.com/whyrusleeping/cbor-gen" ) @@ -87,6 +88,7 @@ type Node struct { // for fetching and storing children store cbor.IpldStore + proto ipld.NodePrototype } // Pointer is an element in a HAMT node's Pointers array, encoded as an IPLD @@ -177,7 +179,7 @@ func NewNode(cs cbor.IpldStore, options ...Option) (*Node, error) { } } - return newNode(cs, cfg.hashFn, cfg.bitWidth), nil + return newNode(cs, cfg.hashFn, cfg.bitWidth, cfg.proto), nil } // Find navigates through the HAMT structure to where key `k` should exist. If @@ -231,13 +233,14 @@ func (n *Node) Delete(ctx context.Context, k string) (bool, error) { } // Constructs a new node value. -func newNode(cs cbor.IpldStore, hashFn HashFunc, bitWidth int) *Node { +func newNode(cs cbor.IpldStore, hashFn HashFunc, bitWidth int, proto ipld.NodePrototype) *Node { nd := &Node{ Bitfield: big.NewInt(0), Pointers: make([]*Pointer, 0), bitWidth: bitWidth, hash: hashFn, store: cs, + proto: proto, } return nd } @@ -276,6 +279,7 @@ func (n *Node) getValue(ctx context.Context, hv *hashBits, k string, cb func(*KV if err != nil { return err } + chnd.proto = n.proto return chnd.getValue(ctx, hv, k, cb) } @@ -446,6 +450,7 @@ func (n *Node) checkSize(ctx context.Context) (uint64, error) { if err != nil { return 0, err } + chnd.proto = n.proto chsize, err := chnd.checkSize(ctx) if err != nil { return 0, err @@ -736,7 +741,7 @@ func (n *Node) modifyValue(ctx context.Context, hv *hashBits, k []byte, v *cbg.D if len(child.KVs) >= bucketSize { // bucket is full, create a child node (shard) with all existing bucket // elements plus the new one and set it in the place of the bucket - sub := newNode(n.store, n.hash, n.bitWidth) + sub := newNode(n.store, n.hash, n.bitWidth, n.proto) hvcopy := &hashBits{b: hv.b, consumed: hv.consumed} if _, err := sub.modifyValue(ctx, hvcopy, k, v, replace); err != nil { return UNMODIFIED, err @@ -817,7 +822,7 @@ func (n *Node) getPointer(i byte) *Pointer { // as cached nodes. func (n *Node) Copy() *Node { // TODO(rvagg): clarify what situations this method is actually useful for. - nn := newNode(n.store, n.hash, n.bitWidth) + nn := newNode(n.store, n.hash, n.bitWidth, n.proto) nn.Bitfield.Set(n.Bitfield) nn.Pointers = make([]*Pointer, len(n.Pointers)) @@ -857,6 +862,7 @@ func (n *Node) ForEach(ctx context.Context, f func(k string, val *cbg.Deferred) if err != nil { return err } + chnd.proto = n.proto if err := chnd.ForEach(ctx, f); err != nil { return err diff --git a/ipld.go b/ipld.go new file mode 100644 index 0000000..e493aac --- /dev/null +++ b/ipld.go @@ -0,0 +1,297 @@ +package hamt + +import ( + "bytes" + "context" + + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/node/basicnode" +) + +var _ ipld.Node = (*Node)(nil) + +func (n *Node) AsBool() (bool, error) { + return false, ipld.ErrWrongKind{} +} + +func (n *Node) AsBytes() ([]byte, error) { + return nil, ipld.ErrWrongKind{} +} + +func (n *Node) AsString() (string, error) { + return "", ipld.ErrWrongKind{} +} + +func (n *Node) AsInt() (int64, error) { + return 0, ipld.ErrWrongKind{} +} + +func (n *Node) AsFloat() (float64, error) { + return 0.0, ipld.ErrWrongKind{} +} + +func (n *Node) AsLink() (ipld.Link, error) { + return nil, ipld.ErrWrongKind{} +} + +func (n *Node) IsAbsent() bool { + return false +} + +func (n *Node) IsNull() bool { + return n.Bitfield == nil +} + +func (n *Node) Length() int64 { + l := int64(0) + for _, p := range n.Pointers { + if p.Link.Defined() { + c, err := p.loadChild(context.Background(), n.store, n.bitWidth, n.hash) + if err != nil { + return -1 + } + c.proto = n.proto + l += c.Length() + } else { + l += int64(len(p.KVs)) + } + } + return int64(l) +} + +func (n *Node) Kind() ipld.Kind { + return ipld.Kind_Map +} + +// LookupByString looks up a child object in this node and returns it. +// The returned Node may be any of the ReprKind: +// a primitive (string, int, etc), a map, a list, or a link. +// +// If the Kind of this Node is not ReprKind_Map, a nil node and an error +// will be returned. +// +// If the key does not exist, a nil node and an error will be returned. +func (n *Node) LookupByString(key string) (ipld.Node, error) { + ok, data, err := n.FindRaw(context.Background(), key) + if err != nil { + return nil, err + } + if !ok { + return nil, ipld.ErrNotExists{} + } + _, val, err := n.realize(key, data) + return val, err +} + +func (n *Node) LookupByNode(key ipld.Node) (ipld.Node, error) { + if key.Kind() == ipld.Kind_String { + s, e := key.AsString() + if e != nil { + return nil, e + } + return n.LookupByString(s) + } else if key.Kind() == ipld.Kind_Bytes { + b, e := key.AsBytes() + if e != nil { + return nil, e + } + return n.LookupByString(string(b)) + } + return nil, ipld.ErrInvalidKey{} +} + +func (n *Node) LookupByIndex(idx int64) (ipld.Node, error) { + return nil, ipld.ErrWrongKind{} +} + +func (n *Node) LookupBySegment(seg ipld.PathSegment) (ipld.Node, error) { + return n.LookupByString(seg.String()) +} + +// MapIterator returns an iterator which yields key-value pairs +// traversing the node. +// If the node kind is anything other than a map, nil will be returned. +// +// The iterator will yield every entry in the map; that is, it +// can be expected that itr.Next will be called node.Length times +// before itr.Done becomes true. +func (n *Node) MapIterator() ipld.MapIterator { + mi := &hmi{ + at: n, + ukv: make([]*KV, 0), + up: n.Pointers, + } + return mi +} + +type hmi struct { + at *Node + ukv []*KV + up []*Pointer + err error +} + +func (mi *hmi) Done() bool { + if len(mi.ukv) == 0 && len(mi.up) == 0 { + return true + } + if len(mi.ukv) > 0 { + return false + } + mi.loadNext() + return mi.Done() +} + +func (mi *hmi) loadNext() { + p := mi.up[0] + mi.up = mi.up[1:] + if p.isShard() { + chld, err := p.loadChild(context.Background(), mi.at.store, mi.at.bitWidth, mi.at.hash) + if err != nil { + mi.err = err + return + } + chld.proto = mi.at.proto + mi.up = append(mi.up, chld.Pointers...) + } else { + mi.ukv = append(mi.ukv, p.KVs...) + } +} + +func (mi *hmi) Next() (ipld.Node, ipld.Node, error) { + if mi.err != nil { + return nil, nil, mi.err + } + // If false, we've ensured at least one entry in mi.ukv + if mi.Done() { + return nil, nil, mi.err + } + + kv := mi.ukv[0] + mi.ukv = mi.ukv[1:] + return mi.at.realize(string(kv.Key), kv.Value.Raw) +} + +func (n *Node) realize(key string, value []byte) (ipld.Node, ipld.Node, error) { + ma, err := n.proto.NewBuilder().BeginMap(0) + if err != nil { + return nil, nil, err + } + + mak := ma.KeyPrototype() + mav := ma.ValuePrototype(key) + + keyBuilder := mak.NewBuilder() + if err := keyBuilder.AssignString(key); err != nil { + return nil, nil, err + } + + valueBuilder := mav.NewBuilder() + if err := dagcbor.Decode(valueBuilder, bytes.NewBuffer(value)); err != nil { + return nil, nil, err + } + return keyBuilder.Build(), valueBuilder.Build(), nil +} + +func (n *Node) ListIterator() ipld.ListIterator { + return nil +} + +func (n *Node) Prototype() ipld.NodePrototype { + return n.proto +} + +func NewTypedHamt(key ipld.NodePrototype, value ipld.NodePrototype) ipld.NodePrototype { + return &hamtProto{key, value} +} + +type hamtProto struct { + k ipld.NodePrototype + v ipld.NodePrototype +} + +func (h *hamtProto) NewBuilder() ipld.NodeBuilder { + return &hamtBuilder{h, nil} +} + +type hamtBuilder struct { + proto *hamtProto + + n *Node +} + +func (h *hamtBuilder) Build() ipld.Node { + return h.n +} + +func (h *hamtBuilder) Reset() { + h.n = nil +} + +func (h *hamtBuilder) BeginMap(sizeHint int64) (ipld.MapAssembler, error) { + return h, nil +} + +func (h *hamtBuilder) AssembleKey() ipld.NodeAssembler { + return nil +} + +func (h *hamtBuilder) AssembleValue() ipld.NodeAssembler { + return nil +} + +func (h *hamtBuilder) AssembleEntry(k string) (ipld.NodeAssembler, error) { + return nil, nil +} + +func (h *hamtBuilder) Finish() error { + return nil +} +func (h *hamtBuilder) KeyPrototype() ipld.NodePrototype { + return h.proto.k +} +func (h *hamtBuilder) ValuePrototype(k string) ipld.NodePrototype { + return h.proto.v +} + +func (h *hamtBuilder) BeginList(sizeHint int64) (ipld.ListAssembler, error) { + return nil, ipld.ErrWrongKind{} +} +func (h *hamtBuilder) AssignNull() error { + return ipld.ErrWrongKind{} +} +func (h *hamtBuilder) AssignBool(bool) error { + return ipld.ErrWrongKind{} +} +func (h *hamtBuilder) AssignInt(int64) error { + return ipld.ErrWrongKind{} +} +func (h *hamtBuilder) AssignFloat(float64) error { + return ipld.ErrWrongKind{} +} +func (h *hamtBuilder) AssignString(string) error { + return ipld.ErrWrongKind{} +} +func (h *hamtBuilder) AssignBytes([]byte) error { + return ipld.ErrWrongKind{} +} +func (h *hamtBuilder) AssignLink(ipld.Link) error { + return ipld.ErrWrongKind{} +} + +func (h *hamtBuilder) AssignNode(n ipld.Node) error { + hn, ok := n.(*Node) + if !ok { + return ErrMalformedHamt + } + h.n = hn + return nil +} + +func (h *hamtBuilder) Prototype() ipld.NodePrototype { + if h.proto == nil { + return basicnode.Prototype__Map{} + } + return h.proto +} diff --git a/options.go b/options.go index dfc09ff..8ff04db 100644 --- a/options.go +++ b/options.go @@ -1,6 +1,10 @@ package hamt -import "fmt" +import ( + "fmt" + + "github.com/ipld/go-ipld-prime" +) const bucketSize = 3 const defaultBitWidth = 8 @@ -8,6 +12,7 @@ const defaultBitWidth = 8 type config struct { bitWidth int hashFn HashFunc + proto ipld.NodePrototype } func defaultConfig() *config { @@ -58,3 +63,12 @@ func UseHashFunction(hash HashFunc) Option { return nil } } + +// WithProto creates a HAMT using a predefined prototype for values, when +// used in an IPLD ADL context. +func WithProto(p ipld.NodePrototype) Option { + return func(c *config) error { + c.proto = p + return nil + } +} diff --git a/reification.go b/reification.go new file mode 100644 index 0000000..ef97f72 --- /dev/null +++ b/reification.go @@ -0,0 +1,169 @@ +package hamt + +import ( + "bytes" + "fmt" + "io" + "math/big" + + block "github.com/ipfs/go-block-format" + cid "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + cbg "github.com/whyrusleeping/cbor-gen" +) + +// Reify looks at an ipld Node and tries to interpret it as a HAMT with default options. +// if successful, it returns the HAMT +func Reify(lnkCtx ipld.LinkContext, maybeHamt ipld.Node, lsys *ipld.LinkSystem) (ipld.Node, error) { + rf := MakeReifier() + return rf(lnkCtx, maybeHamt, lsys) +} + +// MakeReifier allows generation of a Reify function with an explicit hash / bitwidth. +func MakeReifier(opts ...Option) ipld.NodeReifier { + return func(lnkCtx ipld.LinkContext, maybeHamt ipld.Node, lsys *ipld.LinkSystem) (ipld.Node, error) { + if maybeHamt.Kind() != ipld.Kind_List { + return maybeHamt, nil + } + + // per Node layout: + li := maybeHamt.ListIterator() + idx, bitField, err := li.Next() + if err != nil || idx != 0 { + return maybeHamt, nil + } + bitFieldBytes, err := bitField.AsBytes() + if err != nil { + return maybeHamt, nil + } + idx, pointers, err := li.Next() + if err != nil || idx != 1 { + return maybeHamt, nil + } + if pointers.Kind() != ipld.Kind_List { + return maybeHamt, nil + } + if !li.Done() { + return maybeHamt, nil + } + ptrs := make([]*Pointer, 0) + pti := pointers.ListIterator() + for !pti.Done() { + _, p, err := pti.Next() + if err != nil { + return maybeHamt, nil + } + ptr, err := loadPointer(p) + if err != nil { + return maybeHamt, nil + } + ptrs = append(ptrs, ptr) + } + + bs := LSBlockstore{lsys} + store := cbor.NewCborStore(&bs) + bfi := big.NewInt(0).SetBytes(bitFieldBytes) + + cfg := defaultConfig() + for _, o := range opts { + o(cfg) + } + hn := newNode(store, cfg.hashFn, cfg.bitWidth, cfg.proto) + hn.Bitfield = bfi + hn.Pointers = ptrs + + return hn, nil + } +} + +func loadPointer(p ipld.Node) (*Pointer, error) { + if p.Kind() != ipld.Kind_Link { + lnk, err := p.AsLink() + if err != nil { + return nil, fmt.Errorf("pointer union indicates link, but not link") + } + return &Pointer{Link: lnk.(cidlink.Link).Cid}, nil + } else if p.Kind() == ipld.Kind_List { + kvs := make([]*KV, 0) + li := p.ListIterator() + for !li.Done() { + _, kvn, err := li.Next() + if err != nil { + return nil, fmt.Errorf("failed to load list item: %w", err) + } + kv, err := loadKV(kvn) + if err != nil { + return nil, err + } + kvs = append(kvs, kv) + } + return &Pointer{KVs: kvs}, nil + } + return nil, fmt.Errorf("unsupported pointer kind") +} + +func loadKV(n ipld.Node) (*KV, error) { + if n.Kind() != ipld.Kind_List { + return nil, fmt.Errorf("kv should be of kind list") + } + li := n.ListIterator() + _, k, err := li.Next() + if err != nil { + return nil, err + } + kb, err := k.AsBytes() + if err != nil { + return nil, err + } + _, v, err := li.Next() + if err != nil { + return nil, err + } + if !li.Done() { + return nil, fmt.Errorf("kv did not have 2 entries as expected") + } + + buf := bytes.NewBuffer(nil) + if err := dagcbor.Encode(v, buf); err != nil { + return nil, err + } + + return &KV{ + Key: kb, + Value: &cbg.Deferred{Raw: buf.Bytes()}, + }, nil +} + +// LSBlockstore creates a blockstore (get/put) interface over a link system +type LSBlockstore struct { + *ipld.LinkSystem +} + +// Get a raw block of data from a cid using a link system storage read opener +func (l *LSBlockstore) Get(c cid.Cid) (block.Block, error) { + rdr, err := l.StorageReadOpener(linking.LinkContext{}, cidlink.Link{Cid: c}) + if err != nil { + return nil, err + } + bytes, err := io.ReadAll(rdr) + if err != nil { + return nil, err + } + return block.NewBlockWithCid(bytes, c) +} + +// Put a block of data to an underlying store using a link system storage write opener +func (l *LSBlockstore) Put(b block.Block) error { + w, committer, err := l.LinkSystem.StorageWriteOpener(linking.LinkContext{}) + if err != nil { + return err + } + if _, err := w.Write(b.RawData()); err != nil { + return err + } + return committer(cidlink.Link{Cid: b.Cid()}) +}