diff --git a/.github/workflows/build-universal.yaml b/.github/workflows/build-universal.yaml index 0071ebd82..816b3689b 100644 --- a/.github/workflows/build-universal.yaml +++ b/.github/workflows/build-universal.yaml @@ -64,9 +64,7 @@ jobs: if: runner.os == 'macOS' run: | brew install ninja pkg-config automake coreutils libtool autoconf \ - texinfo wget ccache llvm@16 boost jemalloc - env: - HOMEBREW_NO_AUTO_UPDATE: 1 + texinfo wget ccache boost jemalloc cmake - name: Setup Python uses: actions/setup-python@v5 diff --git a/.gitmodules b/.gitmodules index 3b99304a6..f0e34181f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,10 +13,6 @@ [submodule "third-party/cppkafka"] path = third-party/cppkafka url = https://github.com/mfontanini/cppkafka.git -[submodule "third-party/pybind11"] - path = third-party/pybind11 - url = https://github.com/pybind/pybind11.git - branch = stable [submodule "third-party/blst"] path = third-party/blst url = https://github.com/supranational/blst.git diff --git a/Changelog.md b/Changelog.md index b104d10ad..8c6ed697d 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,15 @@ +## 2025.10 Update + +1. [TVM version v12](./doc/GlobalVersions.md): full bounces, new `BTOS` and `HASHBU` instuctions, limit on contract size in masterchain. +2. Optimistic collation/validation: allow nodes to generate and check block candidates before previous block is fully signed (not fully activated yet). +3. Introduced custom block compression algorithm. +4. Overlay improvements: improved overlay discovery on shard configuration update, private externals in custom overlays. +5. Various improvements: session stats, telemetry in fast-sync overlay, earlier block broadcasts, limiting ttl for values in DHT, fixing search by utime in native blockexplorer, faster downloading candidates in validator session, parallelization of storing to cell_db, avoiding touching packfiles on startup. + +Besides the work of the core team, this update is based on the efforts of the Tonstudio team: @hacker-volodya @Shvandre; and @mkiesel (avoiding touching packfiles on startup). + + + ## 2025.07 Accelerator Update Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators) diff --git a/adnl/adnl-ext-client.cpp b/adnl/adnl-ext-client.cpp index 86d259b18..eef5007af 100644 --- a/adnl/adnl-ext-client.cpp +++ b/adnl/adnl-ext-client.cpp @@ -31,6 +31,14 @@ void AdnlExtClientImpl::alarm() { next_create_at_ = td::Timestamp::in(next_alarm_); alarm_timestamp() = next_create_at_; + if (!dst_host_.empty()) { + auto S = dst_addr_.init_host_port(dst_host_); + if (S.is_error()) { + LOG(INFO) << "failed to connect to " << dst_host_ << ": " << S; + return; + } + LOG(DEBUG) << "resolved " << dst_host_ << " -> " << dst_addr_; + } auto fd = td::SocketFd::open(dst_addr_); if (fd.is_error()) { LOG(INFO) << "failed to connect to " << dst_addr_ << ": " << fd.move_as_error(); @@ -166,6 +174,12 @@ td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, td: return td::actor::create_actor("extclient", std::move(dst), dst_addr, std::move(callback)); } +td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, std::string dst_host, + std::unique_ptr callback) { + return td::actor::create_actor("extclient", std::move(dst), std::move(dst_host), + std::move(callback)); +} + td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback) { diff --git a/adnl/adnl-ext-client.h b/adnl/adnl-ext-client.h index 54fae986d..39b9d25b4 100644 --- a/adnl/adnl-ext-client.h +++ b/adnl/adnl-ext-client.h @@ -40,6 +40,8 @@ class AdnlExtClient : public td::actor::Actor { td::Promise promise) = 0; static td::actor::ActorOwn create(AdnlNodeIdFull dst, td::IPAddress dst_addr, std::unique_ptr callback); + static td::actor::ActorOwn create(AdnlNodeIdFull dst, std::string dst_host, + std::unique_ptr callback); static td::actor::ActorOwn create(AdnlNodeIdFull dst, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback); }; diff --git a/adnl/adnl-ext-client.hpp b/adnl/adnl-ext-client.hpp index 1f48e2c49..7556cb100 100644 --- a/adnl/adnl-ext-client.hpp +++ b/adnl/adnl-ext-client.hpp @@ -71,6 +71,9 @@ class AdnlExtClientImpl : public AdnlExtClient { AdnlExtClientImpl(AdnlNodeIdFull dst_id, td::IPAddress dst_addr, std::unique_ptr callback) : dst_(std::move(dst_id)), dst_addr_(dst_addr), callback_(std::move(callback)) { } + AdnlExtClientImpl(AdnlNodeIdFull dst_id, std::string dst_host, std::unique_ptr callback) + : dst_(std::move(dst_id)), dst_host_(std::move(dst_host)), callback_(std::move(callback)) { + } AdnlExtClientImpl(AdnlNodeIdFull dst_id, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback) : dst_(std::move(dst_id)), local_id_(local_id), dst_addr_(dst_addr), callback_(std::move(callback)) { @@ -138,6 +141,7 @@ class AdnlExtClientImpl : public AdnlExtClient { AdnlNodeIdFull dst_; PrivateKey local_id_; td::IPAddress dst_addr_; + std::string dst_host_; std::unique_ptr callback_; diff --git a/assembly/native/build-universal-static.sh b/assembly/native/build-universal-static.sh index b938cd679..eb468cb91 100755 --- a/assembly/native/build-universal-static.sh +++ b/assembly/native/build-universal-static.sh @@ -45,9 +45,10 @@ rm -rf .ninja* CMakeCache.txt CMakeFiles # Detect OS and set compiler # ---------------------- if [[ "$OSTYPE" == "darwin"* ]]; then - echo "Detected macOS" - export CC="$(brew --prefix llvm@16)/bin/clang" - export CXX="$(brew --prefix llvm@16)/bin/clang++" + echo "CC=$(xcrun -find clang)" >> $GITHUB_ENV + echo "CXX=$(xcrun -find clang++)" >> $GITHUB_ENV +# export CC="$(brew --prefix llvm@16)/bin/clang" +# export CXX="$(brew --prefix llvm@16)/bin/clang++" export OPENSSL_LIBS="$OPENSSL_PATH/lib/libcrypto.a" else echo "Detected Linux" diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index 26a6787e1..6919c5c14 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -525,8 +525,8 @@ HttpQueryBlockSearch::HttpQueryBlockSearch(std::map op } if (opts.count("utime") == 1) { try { - seqno_ = static_cast(std::stoull(opts["utime"])); - mode_ = 1; + utime_ = static_cast(std::stoull(opts["utime"])); + mode_ = 4; } catch (...) { error_ = td::Status::Error("cannot parse utime"); return; @@ -1429,10 +1429,10 @@ void HttpQueryStatus::finish_query() { A << "" << static_cast(x->ts_.at_unix()) << ""; } A << "\n"; - for (td::uint32 i = 0; i < results_.ips.size(); i++) { + for (td::uint32 i = 0; i < results_.addrs.size(); i++) { A << ""; - if (results_.ips[i].is_valid()) { - A << "" << results_.ips[i].get_ip_str() << ":" << results_.ips[i].get_port() << ""; + if (!results_.addrs[i].empty()) { + A << "" << results_.addrs[i] << ""; } else { A << "hidden"; } diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index ca50d5266..d21bc465c 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -187,7 +187,7 @@ class CoreActor : public CoreActorInterface { std::mutex queue_mutex_; std::mutex res_mutex_; std::map> results_; - std::vector addrs_; + std::vector addrs_; static CoreActor* instance_; td::actor::ActorId self_id_; @@ -220,7 +220,7 @@ class CoreActor : public CoreActorInterface { } void get_results(td::uint32 max, td::Promise promise) override { RemoteNodeStatusList r; - r.ips = hide_ips_ ? std::vector{addrs_.size()} : addrs_; + r.addrs = hide_ips_ ? std::vector{addrs_.size()} : addrs_; auto it = results_.rbegin(); while (it != results_.rend() && r.results.size() < max) { r.results.push_back(it->second); @@ -445,14 +445,14 @@ class CoreActor : public CoreActorInterface { r_servers.ensure(); servers = r_servers.move_as_ok(); for (const auto& serv : servers) { - addrs_.push_back(serv.addr); + addrs_.push_back(serv.hostname); } } else { if (!remote_addr_.is_valid()) { LOG(FATAL) << "remote addr not set"; } - addrs_.push_back(remote_addr_); servers.push_back(liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_}); + addrs_.push_back(servers.back().hostname); } n_servers_ = servers.size(); client_ = liteclient::ExtClient::create(std::move(servers), make_callback(), true); diff --git a/blockchain-explorer/blockchain-explorer.hpp b/blockchain-explorer/blockchain-explorer.hpp index 1bae362de..abbbde9e9 100644 --- a/blockchain-explorer/blockchain-explorer.hpp +++ b/blockchain-explorer/blockchain-explorer.hpp @@ -59,7 +59,7 @@ class CoreActorInterface : public td::actor::Actor { }; struct RemoteNodeStatusList { - std::vector ips; + std::vector addrs; std::vector> results; }; virtual ~CoreActorInterface() = default; diff --git a/blockchain-indexer/CMakeLists.txt b/blockchain-indexer/CMakeLists.txt index 65a35c93c..2fc348cee 100644 --- a/blockchain-indexer/CMakeLists.txt +++ b/blockchain-indexer/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) if (NOT OPENSSL_FOUND) diff --git a/blockchain-indexer/indexer.cpp b/blockchain-indexer/indexer.cpp index abb3374ec..85017a79a 100644 --- a/blockchain-indexer/indexer.cpp +++ b/blockchain-indexer/indexer.cpp @@ -1224,7 +1224,7 @@ class IndexerWorker : public td::actor::Actor { */ auto in_msg_dict = std::make_unique(vm::load_cell_slice_ref(extra.in_msg_descr), 256, - block::tlb::aug_InMsgDescr); + block::tlb::aug_InMsgDescrDefault); // std::vector in_msgs_json; // while (!in_msg_dict->is_empty()) { @@ -1985,10 +1985,6 @@ class Indexer : public td::actor::Actor { // td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); } - void send_validator_telemetry(ton::PublicKeyHash key, - ton::tl_object_ptr telemetry) override { - } - Callback(td::actor::ActorId id) : id_(id) { } @@ -2268,10 +2264,6 @@ class IndexerSimple : public td::actor::Actor { // td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); } - void send_validator_telemetry(ton::PublicKeyHash key, - ton::tl_object_ptr telemetry) override { - } - Callback(td::actor::ActorId id) : id_(id) { } diff --git a/blockchain-indexer/json-utils.cpp b/blockchain-indexer/json-utils.cpp index 58bbff30c..95e12e442 100644 --- a/blockchain-indexer/json-utils.cpp +++ b/blockchain-indexer/json-utils.cpp @@ -344,7 +344,7 @@ json parse_message(Ref message_any) { {"grams", block::tlb::t_Grams.as_integer(value_cc.grams)->to_dec_string()}, {"extra", value_cc.other->have_refs() ? parse_extra_currency(value_cc.other->prefetch_ref()) : dummy}}; - answer["ihr_fee"] = block::tlb::t_Grams.as_integer(msg.ihr_fee.write())->to_dec_string(); + answer["extra_flags"] = block::tlb::t_Grams.as_integer(msg.extra_flags.write())->to_dec_string(); answer["fwd_fee"] = block::tlb::t_Grams.as_integer(msg.fwd_fee.write())->to_dec_string(); answer["created_lt"] = msg.created_lt; answer["created_at"] = msg.created_at; diff --git a/common/global-version.h b/common/global-version.h index b54a3bdc5..d88a22c32 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -constexpr int SUPPORTED_VERSION = 11; +constexpr int SUPPORTED_VERSION = 12; } diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 457816abf..1f2d81f27 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -280,9 +280,6 @@ class HardforkCreator : public td::actor::Actor { void new_key_block(ton::validator::BlockHandle handle) override { } - void send_validator_telemetry(ton::PublicKeyHash key, - ton::tl_object_ptr telemetry) override { - } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 8cd4b7cad..cfc4d7a42 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -14,6 +14,7 @@ set(TON_CRYPTO_CORE_SOURCE openssl/residue.cpp openssl/rand.cpp vm/boc.cpp + vm/boc-compression.cpp vm/large-boc-serializer.cpp tl/tlblib.cpp @@ -109,6 +110,7 @@ set(TON_CRYPTO_SOURCE vm/arithops.h vm/atom.h vm/boc.h + vm/boc-compression.h vm/boc-writers.h vm/box.hpp vm/cellops.h diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 71ec2bcf5..9443b69f3 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -665,7 +665,7 @@ bool CommonMsgInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const && t_MsgAddressInt.validate_skip(ops, cs, weak) // src && t_MsgAddressInt.validate_skip(ops, cs, weak) // dest && t_CurrencyCollection.validate_skip(ops, cs, weak) // value - && t_Grams.validate_skip(ops, cs, weak) // ihr_fee + && t_Grams.validate_skip(ops, cs, weak) // extra_flags && t_Grams.validate_skip(ops, cs, weak) // fwd_fee && cs.advance(64 + 32); // created_lt:uint64 created_at:uint32 case ext_in_msg_info: @@ -684,7 +684,7 @@ bool CommonMsgInfo::unpack(vm::CellSlice& cs, CommonMsgInfo::Record_int_msg_info return get_tag(cs) == int_msg_info && cs.advance(1) && cs.fetch_bool_to(data.ihr_disabled) && cs.fetch_bool_to(data.bounce) && cs.fetch_bool_to(data.bounced) && t_MsgAddressInt.fetch_to(cs, data.src) && t_MsgAddressInt.fetch_to(cs, data.dest) && t_CurrencyCollection.fetch_to(cs, data.value) && - t_Grams.fetch_to(cs, data.ihr_fee) && t_Grams.fetch_to(cs, data.fwd_fee) && + t_Grams.fetch_to(cs, data.extra_flags) && t_Grams.fetch_to(cs, data.fwd_fee) && cs.fetch_uint_to(64, data.created_lt) && cs.fetch_uint_to(32, data.created_at); } @@ -696,7 +696,7 @@ bool CommonMsgInfo::skip(vm::CellSlice& cs) const { && t_MsgAddressInt.skip(cs) // src && t_MsgAddressInt.skip(cs) // dest && t_CurrencyCollection.skip(cs) // value - && t_Grams.skip(cs) // ihr_fee + && t_Grams.skip(cs) // extra_flags && t_Grams.skip(cs) // fwd_fee && cs.advance(64 + 32); // created_lt:uint64 created_at:uint32 case ext_in_msg_info: @@ -718,7 +718,7 @@ bool CommonMsgInfo::get_created_lt(vm::CellSlice& cs, unsigned long long& create && t_MsgAddressInt.skip(cs) // src && t_MsgAddressInt.skip(cs) // dest && t_CurrencyCollection.skip(cs) // value - && t_Grams.skip(cs) // ihr_fee + && t_Grams.skip(cs) // extra_flags && t_Grams.skip(cs) // fwd_fee && cs.fetch_ulong_bool(64, created_lt) // created_lt:uint64 && cs.advance(32); // created_at:uint32 @@ -1843,26 +1843,18 @@ bool InMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return false; } -bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { +static td::RefInt256 get_ihr_fee(const CommonMsgInfo::Record_int_msg_info &info, int global_version) { + // Legacy: extra_flags was previously ihr_fee + return global_version >= 12 ? td::zero_refint() : t_Grams.as_integer(std::move(info.extra_flags)); +} + +bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const { int tag = get_tag(cs); switch (tag) { case msg_import_ext: // inbound external message return t_ImportFees.null_value(cb); // external messages have no value and no import fees case msg_import_ihr: // IHR-forwarded internal message to its final destination - if (cs.advance(3) && cs.size_refs() >= 3) { - auto msg_cs = load_cell_slice(cs.fetch_ref()); - CommonMsgInfo::Record_int_msg_info msg_info; - td::RefInt256 ihr_fee; - vm::CellBuilder aux; - // sort of Prolog-style in C++ - return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && - cs.fetch_ref().not_null() && (ihr_fee = t_Grams.as_integer_skip(cs)).not_null() && - cs.fetch_ref().not_null() && !cmp(ihr_fee, t_Grams.as_integer(*msg_info.ihr_fee)) && - cb.append_cellslice_bool(msg_info.ihr_fee) // fees_collected := ihr_fee - && aux.append_cellslice_bool(msg_info.ihr_fee) && t_ExtraCurrencyCollection.null_value(aux) && - t_CurrencyCollection.add_values(cb, aux.as_cellslice_ref().write(), - msg_info.value.write()); // value_imported := ihr_fee + value - } + // IHR is not implemented return false; case msg_import_imm: // internal message re-imported from this very block if (cs.advance(3) && cs.size_refs() >= 2) { @@ -1888,7 +1880,7 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && cb.append_cellslice_bool(in_msg.fwd_fee_remaining) // fees_collected := fwd_fee_remaining && t_Grams.as_integer_skip_to(msg_info.value.write(), value_grams) && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool( msg_info.value.write()); // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining @@ -1911,7 +1903,7 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && t_Grams.store_integer_ref(cb, std::move(transit_fee)) // fees_collected := transit_fees && t_Grams.as_integer_skip_to(msg_info.value.write(), value_grams) && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool( msg_info.value.write()); // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining @@ -1941,8 +1933,8 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { const InMsg t_InMsg; -const Aug_InMsgDescr aug_InMsgDescr; -const InMsgDescr t_InMsgDescr; +const Aug_InMsgDescr aug_InMsgDescrDefault(ton::SUPPORTED_VERSION); +const InMsgDescr t_InMsgDescrDefault(ton::SUPPORTED_VERSION); bool OutMsg::skip(vm::CellSlice& cs) const { switch (get_tag(cs)) { @@ -2038,7 +2030,7 @@ bool OutMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return false; } -bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { +bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const { auto tag = get_tag(cs); switch (tag) { case msg_export_ext: // external outbound message carries no value @@ -2071,7 +2063,7 @@ bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { td::RefInt256 value_grams, ihr_fee, fwd_fee_remaining; return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && (value_grams = t_Grams.as_integer_skip(msg_info.value.write())).not_null() && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && (fwd_fee_remaining = t_Grams.as_integer(out_msg.fwd_fee_remaining)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool(std::move(msg_info.value)); @@ -2112,8 +2104,8 @@ bool OutMsg::get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) c const OutMsg t_OutMsg; -const Aug_OutMsgDescr aug_OutMsgDescr; -const OutMsgDescr t_OutMsgDescr; +const Aug_OutMsgDescr aug_OutMsgDescrDefault(ton::SUPPORTED_VERSION); +const OutMsgDescr t_OutMsgDescrDefault(ton::SUPPORTED_VERSION); bool EnqueuedMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return cs.advance(64) && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak); diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index fd17c6579..2eb8c5d77 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -405,7 +405,7 @@ struct CommonMsgInfo final : TLB_Complex { struct CommonMsgInfo::Record_int_msg_info { bool ihr_disabled, bounce, bounced; - Ref src, dest, value, ihr_fee, fwd_fee; + Ref src, dest, value, extra_flags, fwd_fee; unsigned long long created_lt; unsigned created_at; }; @@ -604,7 +604,7 @@ extern const Aug_ShardAccounts aug_ShardAccounts; struct ShardAccounts final : TLB_Complex { HashmapAugE dict_type; - ShardAccounts() : dict_type(256, aug_ShardAccounts){}; + ShardAccounts() : dict_type(256, aug_ShardAccounts) {}; bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -814,7 +814,7 @@ struct InMsg final : TLB_Complex { } return (int)cs.prefetch_ulong(5) - 0b00100 + 8; } - bool get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const; + bool get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const; }; extern const InMsg t_InMsg; @@ -844,7 +844,7 @@ struct OutMsg final : TLB_Complex { } return t; } - bool get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const; + bool get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const; bool get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) const; }; @@ -853,18 +853,22 @@ extern const OutMsg t_OutMsg; // next: InMsgDescr, OutMsgDescr, OutMsgQueue, and their augmentations struct Aug_InMsgDescr final : AugmentationCheckData { - Aug_InMsgDescr() : AugmentationCheckData(t_InMsg, t_ImportFees) { + explicit Aug_InMsgDescr(int global_version) + : AugmentationCheckData(t_InMsg, t_ImportFees), global_version(global_version) { } bool eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const override { - return t_InMsg.get_import_fees(cb, cs); + return t_InMsg.get_import_fees(cb, cs, global_version); } + int global_version; }; -extern const Aug_InMsgDescr aug_InMsgDescr; +extern const Aug_InMsgDescr aug_InMsgDescrDefault; struct InMsgDescr final : TLB_Complex { + Aug_InMsgDescr aug; HashmapAugE dict_type; - InMsgDescr() : dict_type(256, aug_InMsgDescr){}; + explicit InMsgDescr(int global_version) : aug(global_version), dict_type(256, aug) { + } bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -873,21 +877,25 @@ struct InMsgDescr final : TLB_Complex { } }; -extern const InMsgDescr t_InMsgDescr; +extern const InMsgDescr t_InMsgDescrDefault; struct Aug_OutMsgDescr final : AugmentationCheckData { - Aug_OutMsgDescr() : AugmentationCheckData(t_OutMsg, t_CurrencyCollection) { + explicit Aug_OutMsgDescr(int global_version) + : AugmentationCheckData(t_OutMsg, t_CurrencyCollection), global_version(global_version) { } bool eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const override { - return t_OutMsg.get_export_value(cb, cs); + return t_OutMsg.get_export_value(cb, cs, global_version); } + int global_version; }; -extern const Aug_OutMsgDescr aug_OutMsgDescr; +extern const Aug_OutMsgDescr aug_OutMsgDescrDefault; struct OutMsgDescr final : TLB_Complex { + Aug_OutMsgDescr aug; HashmapAugE dict_type; - OutMsgDescr() : dict_type(256, aug_OutMsgDescr){}; + explicit OutMsgDescr(int global_version) : aug(global_version), dict_type(256, aug) { + } bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -896,7 +904,7 @@ struct OutMsgDescr final : TLB_Complex { } }; -extern const OutMsgDescr t_OutMsgDescr; +extern const OutMsgDescr t_OutMsgDescrDefault; struct EnqueuedMsg final : TLB_Complex { int get_size(const vm::CellSlice& cs) const override { @@ -935,7 +943,7 @@ extern const Aug_DispatchQueue aug_DispatchQueue; struct OutMsgQueue final : TLB_Complex { HashmapAugE dict_type; - OutMsgQueue() : dict_type(32 + 64 + 256, aug_OutMsgQueue){}; + OutMsgQueue() : dict_type(32 + 64 + 256, aug_OutMsgQueue) {}; bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -1138,8 +1146,8 @@ struct Aug_ShardFees final : AugmentationCheckData { extern const Aug_ShardFees aug_ShardFees; // Validate dict of libraries in message: used when sending and receiving message -bool validate_message_libs(const td::Ref &cell); -bool validate_message_relaxed_libs(const td::Ref &cell); +bool validate_message_libs(const td::Ref& cell); +bool validate_message_relaxed_libs(const td::Ref& cell); } // namespace tlb } // namespace block diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 7f0368d02..9f05ed70e 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -125,7 +125,7 @@ currencies$_ grams:Grams other:ExtraCurrencyCollection // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool src:MsgAddressInt dest:MsgAddressInt - value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfo; ext_in_msg_info$10 src:MsgAddressExt dest:MsgAddressInt import_fee:Grams = CommonMsgInfo; @@ -134,7 +134,7 @@ ext_out_msg_info$11 src:MsgAddressInt dest:MsgAddressExt int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool src:MsgAddress dest:MsgAddressInt - value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; @@ -169,6 +169,16 @@ message$_ {X:Type} info:CommonMsgInfoRelaxed _ (Message Any) = MessageAny; + +_ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; +_ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; +new_bounce_body#fffffffe + original_body:^Cell + original_info:^NewBounceOriginalInfo + bounced_by_phase:uint8 exit_code:int32 + compute_phase:(Maybe NewBounceComputePhaseInfo) + = NewBounceBody; + // interm_addr_regular$0 use_dest_bits:(#<= 96) = IntermediateAddress; @@ -804,9 +814,9 @@ _ MisbehaviourPunishmentConfig = ConfigParam 40; size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 - max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 + max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_mc_acc_state_cells:uint32 max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 - max_acc_fixed_prefix_length:uint8 = SizeLimitsConfig; + max_acc_fixed_prefix_length:uint8 acc_state_cells_for_storage_dict:uint32 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; // key is [ wc:int32 addr:uint256 ] diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp index 4ddd701eb..9c8a85e92 100644 --- a/crypto/block/check-proof.cpp +++ b/crypto/block/check-proof.cpp @@ -492,7 +492,7 @@ td::Status BlockProofLink::validate(td::uint32* save_utime) const { return td::Status::Error("BlockProofLink contains a state proof for "s + from.to_str() + " with incorrect root hash"); } - TRY_RESULT(config, block::ConfigInfo::extract_config(vstate_root, block::ConfigInfo::needPrevBlocks)); + TRY_RESULT(config, block::ConfigInfo::extract_config(vstate_root, from, block::ConfigInfo::needPrevBlocks)); if (!config->check_old_mc_block_id(to, true)) { return td::Status::Error("cannot check that "s + to.to_str() + " is indeed a previous masterchain block of " + from.to_str() + " using the presented Merkle proof of masterchain state"); diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 35c3df005..d9a67b0be 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -92,22 +92,16 @@ td::Result> Config::extract_from_state(Ref mc_ return unpack_config(std::move(extra.config), mode); } -td::Result> ConfigInfo::extract_config(std::shared_ptr static_boc, - int mode) { - TRY_RESULT(rc, static_boc->get_root_count()); - if (rc != 1) { - return td::Status::Error(-668, "Masterchain state BoC is invalid"); - } - TRY_RESULT(root, static_boc->get_root_cell(0)); - return extract_config(std::move(root), mode); -} - -td::Result> ConfigInfo::extract_config(Ref mc_state_root, int mode) { +td::Result> ConfigInfo::extract_config(Ref mc_state_root, + ton::BlockIdExt mc_block_id, int mode) { if (mc_state_root.is_null()) { return td::Status::Error("configuration state root cell is null"); } auto config = std::unique_ptr{new ConfigInfo(std::move(mc_state_root), mode)}; TRY_STATUS(config->unpack_wrapped()); + if (!config->set_block_id_ext(mc_block_id)) { + return td::Status::Error("failed to set mc block id"); + } return std::move(config); } @@ -2040,12 +2034,13 @@ td::Result Config::do_get_size_limits_config(td::Ref compute_validator_set_cc(ton::ShardIdFull shard, ton::UnixTime time, ton::CatchainSeqno* cc_seqno_delta = nullptr) const; td::Result> get_prev_blocks_info() const; - static td::Result> extract_config(std::shared_ptr static_boc, - int mode = 0); - static td::Result> extract_config(Ref mc_state_root, int mode = 0); + static td::Result> extract_config(Ref mc_state_root, + ton::BlockIdExt mc_block_id, int mode = 0); private: ConfigInfo(Ref mc_state_root, int _mode = 0); diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 002d6cce5..d3fbd26aa 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -118,7 +118,7 @@ bool Account::set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr) { /** * Sets the length of anycast prefix length in the account address. * - * @param new_length The new rewrite lingth. + * @param new_length The new rewrite length. * * @returns True if the length was successfully set, False otherwise. */ @@ -924,7 +924,16 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* } bounce_enabled = in_msg_info.bounce; in_msg_type = 1; - td::RefInt256 ihr_fee = block::tlb::t_Grams.as_integer(in_msg_info.ihr_fee); + td::RefInt256 ihr_fee; + if (cfg->global_version >= 12) { + ihr_fee = td::zero_refint(); + td::RefInt256 extra_flags = tlb::t_Grams.as_integer(in_msg_info.extra_flags); + new_bounce_format = extra_flags->get_bit(0); + new_bounce_format_full_body = extra_flags->get_bit(1); + } else { + // Legacy: extra_flags was previously ihr_fee + ihr_fee = tlb::t_Grams.as_integer(in_msg_info.extra_flags); + } if (ihr_delivered) { in_fwd_fee = std::move(ihr_fee); } else { @@ -1329,7 +1338,7 @@ namespace transaction { * Checks if it is required to increase gas_limit (from GasLimitsPrices config) for the transaction * * In January 2024 a highload wallet of @wallet Telegram bot in mainnet was stuck because current gas limit (1M) is - * not enough to clean up old queires, thus locking funds inside. + * not enough to clean up old queries, thus locking funds inside. * See comment in crypto/smartcont/highload-wallet-v2-code.fc for details on why this happened. * Account address: EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu * It was proposed to validators to increase gas limit for this account to 70M for a limited amount @@ -1692,7 +1701,7 @@ bool Transaction::unpack_msg_state(const ComputePhaseConfig& cfg, bool lib_only, if (forbid_public_libs) { size_limits.max_acc_public_libraries = 0; } - auto S = check_state_limits(size_limits, false); + auto S = check_state_limits(size_limits, cfg.global_version, false); if (S.is_error()) { LOG(DEBUG) << "Cannot unpack msg state: " << S.move_as_error(); new_code = old_code; @@ -1762,12 +1771,12 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom ComputePhase& cp = *compute_phase; CHECK(cp.precompiled_gas_usage); td::uint64 gas_usage = cp.precompiled_gas_usage.value(); - td::Timer timer; + td::RealCpuTimer timer; auto result = impl.run(my_addr, now, start_lt, balance, new_data, *in_msg_body, in_msg, msg_balance_remaining, in_msg_extern, compute_vm_libraries(cfg), cfg.global_version, cfg.max_vm_data_depth, new_code, cfg.unpacked_config_tuple, due_payment.not_null() ? due_payment : td::zero_refint(), gas_usage); - double elapsed = timer.elapsed(); + time_tvm = timer.elapsed_both(); cp.vm_init_state_hash = td::Bits256::zero(); cp.exit_code = result.exit_code; cp.out_of_gas = false; @@ -1778,7 +1787,7 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom cp.success = (cp.accepted && result.committed); LOG(INFO) << "Running precompiled smart contract " << impl.get_name() << ": exit_code=" << result.exit_code << " accepted=" << result.accepted << " success=" << cp.success << " gas_used=" << gas_usage - << " time=" << elapsed << "s"; + << " time=" << time_tvm.real << "s cpu_time=" << time_tvm.cpu; if (cp.accepted & use_msg_state) { was_activated = true; acc_status = Account::acc_active; @@ -1786,7 +1795,7 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom if (cfg.with_vm_log) { cp.vm_log = PSTRING() << "Running precompiled smart contract " << impl.get_name() << ": exit_code=" << result.exit_code << " accepted=" << result.accepted - << " success=" << cp.success << " gas_used=" << gas_usage << " time=" << elapsed << "s"; + << " success=" << cp.success << " gas_used=" << gas_usage << " time=" << time_tvm.real << "s"; } if (cp.success) { cp.new_data = impl.get_c4(); @@ -1995,9 +2004,9 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { LOG(DEBUG) << "starting VM"; cp.vm_init_state_hash = vm.get_state_hash(); - td::Timer timer; + td::RealCpuTimer timer; cp.exit_code = ~vm.run(); - double elapsed = timer.elapsed(); + time_tvm = timer.elapsed_both(); LOG(DEBUG) << "VM terminated with exit code " << cp.exit_code; cp.out_of_gas = (cp.exit_code == ~(int)vm::Excno::out_of_gas); cp.vm_final_state_hash = vm.get_final_state_hash(cp.exit_code); @@ -2022,8 +2031,8 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { } LOG(INFO) << "steps: " << vm.get_steps_count() << " gas: used=" << gas.gas_consumed() << ", max=" << gas.gas_max << ", limit=" << gas.gas_limit << ", credit=" << gas.gas_credit; - LOG(DEBUG) << "out_of_gas=" << cp.out_of_gas << ", accepted=" << cp.accepted << ", success=" << cp.success - << ", time=" << elapsed << "s"; + LOG(INFO) << "out_of_gas=" << cp.out_of_gas << ", accepted=" << cp.accepted << ", success=" << cp.success + << ", time=" << time_tvm.real << "s, cpu_time=" << time_tvm.cpu; if (logger != nullptr) { cp.vm_log = logger->get_log(); } @@ -2098,7 +2107,7 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { if (account.is_special) { return 1; } - auto S = check_state_limits(cfg.size_limits); + auto S = check_state_limits(cfg.size_limits, cfg.global_version); if (S.is_error()) { if (S.code() != AccountStorageStat::errorcode_limits_exceeded) { LOG(ERROR) << "Account storage stat error: " << S.move_as_error(); @@ -2729,8 +2738,8 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (cfg.disable_custom_fess) { fwd_fee = ihr_fee = td::zero_refint(); } else { - fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); - ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + fwd_fee = tlb::t_Grams.as_integer(info.fwd_fee); + ihr_fee = cfg.global_version >= 12 ? td::zero_refint() : tlb::t_Grams.as_integer(info.extra_flags); } if (cfg.disable_ihr_flag) { info.ihr_disabled = true; @@ -2968,7 +2977,9 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, // re-pack message value CHECK(req.pack_to(info.value)); CHECK(block::tlb::t_Grams.pack_integer(info.fwd_fee, fwd_fee_remain)); - CHECK(block::tlb::t_Grams.pack_integer(info.ihr_fee, ihr_fee)); + if (cfg.global_version < 12) { + CHECK(block::tlb::t_Grams.pack_integer(info.extra_flags, ihr_fee)); + } // serialize message CHECK(tlb::csr_pack(msg.info, info)); @@ -3227,6 +3238,7 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * This function is not called for special accounts. * * @param size_limits The size limits configuration. + * @param global_version Global version (ConfigParam 8). * @param is_account_stat Store storage stat in the Transaction's AccountStorageStat. * * @returns A `td::Status` indicating the result of the check. @@ -3234,7 +3246,8 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * - If the state limits exceed the maximum allowed range, returns an error with AccountStorageStat::errorcode_limits_exceeded code. * - If an error occurred during storage stat calculation, returns other error. */ -td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool is_account_stat) { +td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, int global_version, + bool is_account_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { return a.is_null() || b.is_null() ? a.is_null() == b.is_null() : a->get_hash() == b->get_hash(); }; @@ -3248,7 +3261,13 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, } { TD_PERF_COUNTER(transaction_storage_stat_a); - td::Timer timer; + td::RealCpuTimer timer; + SCOPE_EXIT { + LOG_IF(INFO, timer.elapsed_real() > 0.1) << "Compute used storage (1) took " << timer.elapsed_real() << "s"; + if (is_account_stat) { + time_storage_stat += timer.elapsed_both(); + } + }; if (is_account_stat && compute_phase) { storage_stat.add_hint(compute_phase->vm_loaded_cells); } @@ -3260,18 +3279,14 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, storage_stat_updates.push_back(new_library); } TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); - if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s"; - } } - if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells || - storage_stat.get_total_bits() > size_limits.max_acc_state_bits) { + td::uint32 max_cells = account.is_masterchain() && global_version >= 12 ? size_limits.max_mc_acc_state_cells + : size_limits.max_acc_state_cells; + if (storage_stat.get_total_cells() > max_cells) { return td::Status::Error(AccountStorageStat::errorcode_limits_exceeded, PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() - << ", bits=" << storage_stat.get_total_bits() - << " (max cells=" << size_limits.max_acc_state_cells - << ", max bits=" << size_limits.max_acc_state_bits << ")"); + << " (max cells=" << max_cells << ")"); } if (account.is_masterchain() && !cell_equal(account.library, new_library)) { auto libraries_count = get_public_libraries_count(new_library); @@ -3301,8 +3316,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { } bounce_phase = std::make_unique(); BouncePhase& bp = *bounce_phase; - block::gen::Message::Record msg; - block::gen::CommonMsgInfo::Record_int_msg_info info; + gen::Message::Record msg; + gen::CommonMsgInfo::Record_int_msg_info info; auto cs = vm::load_cell_slice(in_msg); if (!(tlb::unpack(cs, info) && gen::t_Maybe_Either_StateInit_Ref_StateInit.skip(cs) && cs.have(1) && cs.have_refs((int)cs.prefetch_ulong(1)))) { @@ -3312,6 +3327,44 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { if (cs.fetch_ulong(1)) { cs = vm::load_cell_slice(cs.prefetch_ref()); } + + vm::CellBuilder body; + if (new_bounce_format) { + body.store_long(0xfffffffeU, 32); // new_bounce_body#fffffffe + if (new_bounce_format_full_body) { // original_body:^Cell + body.store_ref(vm::CellBuilder().append_cellslice(in_msg_body).finalize_novm()); + } else { + body.store_ref(vm::CellBuilder().store_bits(in_msg_body->as_bitslice()).finalize_novm()); + } + body.store_ref(vm::CellBuilder() + .append_cellslice(in_msg_info.value) // value:CurrencyCollection + .store_long(in_msg_info.created_lt, 64) // created_lt:uint64 + .store_long(in_msg_info.created_at, 32) // created_at:uint32 + .finalize_novm()); // original_info:^NewBounceOriginalInfo + if (compute_phase->skip_reason != ComputePhase::sk_none) { + body.store_long(0, 8); // bounced_by_phase:uint8 + body.store_long(-compute_phase->skip_reason, 32); // exit_code:int32 + } else if (!compute_phase->success) { + body.store_long(1, 8); // bounced_by_phase:uint8 + body.store_long(compute_phase->exit_code, 32); // exit_code:int32 + } else { + body.store_long(2, 8); // bounced_by_phase:uint8 + body.store_long(action_phase->result_code, 32); // exit_code:int32 + } + // compute_phase:(Maybe NewBounceComputePhaseInfo) + if (compute_phase->skip_reason != ComputePhase::sk_none) { + body.store_long(0, 1); + } else { + body.store_long(1, 1); + body.store_long(compute_phase->gas_used, 32); // gas_used:uint32 + body.store_long(compute_phase->vm_steps, 32); // vm_steps:uint32 + } + } else if (cfg.bounce_msg_body) { + int body_bits = std::min((int)cs.size(), cfg.bounce_msg_body); + body.store_long_bool(-1, 32); // 0xffffffff tag + body.append_bitslice(cs.prefetch_bits(body_bits)); // truncated message body + } + info.ihr_disabled = true; info.bounce = false; info.bounced = true; @@ -3327,7 +3380,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { // compute size of message vm::CellStorageStat sstat; // for message size // preliminary storage estimation of the resulting message - sstat.compute_used_storage(info.value->prefetch_ref()); + sstat.add_used_storage(info.value->prefetch_ref()); + sstat.add_used_storage(body.get_refs()); bp.msg_bits = sstat.bits; bp.msg_cells = sstat.cells; // compute forwarding fees @@ -3363,26 +3417,17 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { && cb.append_cellslice_bool(info.src) // src:MsgAddressInt && cb.append_cellslice_bool(info.dest) // dest:MsgAddressInt && msg_balance.store(cb) // value:CurrencyCollection - && block::tlb::t_Grams.store_long(cb, 0) // ihr_fee:Grams + && block::tlb::t_Grams.store_long(cb, 0) // extra_flags:(VarUInteger 16) && block::tlb::t_Grams.store_long(cb, bp.fwd_fees) // fwd_fee:Grams && cb.store_long_bool(info.created_lt, 64) // created_lt:uint64 && cb.store_long_bool(info.created_at, 32) // created_at:uint32 && cb.store_bool_bool(false)); // init:(Maybe ...) - if (cfg.bounce_msg_body) { - int body_bits = std::min((int)cs.size(), cfg.bounce_msg_body); - if (cb.remaining_bits() >= body_bits + 33u) { - CHECK(cb.store_bool_bool(false) // body:(Either X ^X) -> left X - && cb.store_long_bool(-1, 32) // int = -1 ("message type") - && cb.append_bitslice(cs.prefetch_bits(body_bits))); // truncated message body - } else { - vm::CellBuilder cb2; - CHECK(cb.store_bool_bool(true) // body:(Either X ^X) -> right ^X - && cb2.store_long_bool(-1, 32) // int = -1 ("message type") - && cb2.append_bitslice(cs.prefetch_bits(body_bits)) // truncated message body - && cb.store_builder_ref_bool(std::move(cb2))); // ^X - } + if (cb.can_extend_by(1 + body.size(), body.size_refs())) { + // body:(Either X ^X) -> left X + CHECK(cb.store_bool_bool(false) && cb.append_builder_bool(body)); } else { - CHECK(cb.store_bool_bool(false)); // body:(Either ..) + // body:(Either X ^X) -> right ^X + CHECK(cb.store_bool_bool(true) && cb.store_builder_ref_bool(std::move(body))); } CHECK(cb.finalize_to(bp.out_msg)); if (verbosity > 2) { @@ -3563,9 +3608,11 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { auto roots = new_storage_for_stat->prefetch_all_refs(); storage_stat_updates.insert(storage_stat_updates.end(), roots.begin(), roots.end()); { + td::RealCpuTimer timer; StorageStatCalculationContext context{true}; StorageStatCalculationContext::Guard guard{&context}; td::Status S = stats.replace_roots(roots); + time_storage_stat += timer.elapsed_both(); if (S.is_error()) { LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); return false; @@ -3574,8 +3621,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { // Root of AccountStorage is not counted in AccountStorageStat new_storage_used.cells = stats.get_total_cells() + 1; new_storage_used.bits = stats.get_total_bits() + new_storage_for_stat->size(); - // TODO: think about this limit (25) - if (store_storage_dict_hash && new_storage_used.cells > 25) { + if (store_storage_dict_hash && new_storage_used.cells >= cfg.size_limits.acc_state_cells_for_storage_dict) { auto r_hash = stats.get_dict_hash(); if (r_hash.is_error()) { LOG(ERROR) << "Cannot compute storage dict hash for account " << account.addr.to_hex() << ": " @@ -4231,11 +4277,13 @@ td::Status FetchConfigParams::fetch_config_params( action_phase_cfg->extra_currency_v2 = config.get_global_version() >= 10; action_phase_cfg->disable_anycast = config.get_global_version() >= 10; action_phase_cfg->disable_ihr_flag = config.get_global_version() >= 11; + action_phase_cfg->global_version = config.get_global_version(); } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; serialize_cfg->disable_anycast = config.get_global_version() >= 10; serialize_cfg->store_storage_dict_hash = config.get_global_version() >= 11; + serialize_cfg->size_limits = size_limits; } { // fetch block_grams_created diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index c91bd6bc5..8da7df3a5 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -178,6 +178,7 @@ struct ActionPhaseConfig { bool disable_ihr_flag{false}; td::optional mc_blackhole_addr; bool disable_anycast{false}; + int global_version = 0; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { return is_masterchain ? fwd_mc : fwd_std; } @@ -187,6 +188,7 @@ struct SerializeConfig { bool extra_currency_v2{false}; bool disable_anycast{false}; bool store_storage_dict_hash{false}; + SizeLimitsConfig size_limits; }; struct CreditPhase { @@ -195,7 +197,7 @@ struct CreditPhase { }; struct ComputePhase { - enum { sk_none, sk_no_state, sk_bad_state, sk_no_gas, sk_suspended }; + enum { sk_none = 0, sk_no_state = 1, sk_bad_state = 2, sk_no_gas = 3, sk_suspended = 4 }; int skip_reason{sk_none}; bool success{false}; bool msg_state_used{false}; @@ -357,6 +359,8 @@ struct Transaction { bool bounce_enabled{false}; bool in_msg_extern{false}; gen::CommonMsgInfo::Record_int_msg_info in_msg_info; + bool new_bounce_format{false}; + bool new_bounce_format_full_body{false}; bool use_msg_state{false}; bool is_first{false}; bool orig_addr_rewrite_set{false}; @@ -400,6 +404,7 @@ struct Transaction { td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; std::vector> storage_stat_updates; + td::RealCpuTimer::Time time_tvm, time_storage_stat; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); bool unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg); @@ -413,7 +418,7 @@ struct Transaction { bool run_precompiled_contract(const ComputePhaseConfig& cfg, precompiled::PrecompiledSmartContract& precompiled); bool prepare_compute_phase(const ComputePhaseConfig& cfg); bool prepare_action_phase(const ActionPhaseConfig& cfg); - td::Status check_state_limits(const SizeLimitsConfig& size_limits, bool is_account_stat = true); + td::Status check_state_limits(const SizeLimitsConfig& size_limits, int global_version, bool is_account_stat = true); bool prepare_bounce_phase(const ActionPhaseConfig& cfg); bool compute_state(const SerializeConfig& cfg); bool serialize(const SerializeConfig& cfg); diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 38ef69440..082c2436c 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -697,6 +697,7 @@ x{CF3F} @Defop BCHKBITREFSQ x{CF40} @Defop STZEROES x{CF41} @Defop STONES x{CF42} @Defop STSAME +x{CF50} @Defop BTOS { tuck sbitrefs swap 22 + swap @havebitrefs not { swap PUSHSLICE STSLICER } { over sbitrefs 2dup 57 3 2x<= @@ -1390,6 +1391,7 @@ x{F912} @Defop ECRECOVER x{F913} @Defop SECP256K1_XONLY_PUBKEY_TWEAK_ADD x{F914} @Defop P256_CHKSIGNU x{F915} @Defop P256_CHKSIGNS +x{F916} @Defop HASHBU x{F920} @Defop RIST255_FROMHASH x{F921} @Defop RIST255_VALIDATE diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 7d21dd7d9..e0a81bfb9 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,14 +1,132 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. -// More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 1.0 +// More specific functions are required to be imported explicitly, like "@stdlib/gas-payments". +tolk 1.1 -/// In Tolk v1.x there would be a type `map`. -/// Currently, working with dictionaries is still low-level, with raw cells. -/// But just for clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. -/// Every dictionary object can be null. TVM NULL is essentially "empty dictionary". +/** + Built-in types. + */ + +/// `int` is the primitive 257-bit signed integer type. +type int = builtin + +/// `bool` is a classic boolean type, which can hold only two values: `true` and `false`. +/// At the TVM (TON virtual machine) level, it's an integer -1 or 0. +/// Note: `boolVar as int` is possible, but remember, that true is -1, not 1! +type bool = builtin + +/// `cell` is a data structure, which can hold of up to 1023 bits (not bytes!) +/// and up to 4 references (refs) to other cells. +/// Both contract code and contract state are represented by a tree of cells. +/// [See docs](https://docs.ton.org/v3/documentation/data-formats/tlb/cell-boc) +type cell = builtin + +/// `slice` is a "cell opened for reading". +/// When you call [cell.beginParse], you get a `slice`, from which you can load binary data +/// or high-level structures with [T.fromSlice]. +type slice = builtin + +/// `builder` is a "cell at the stage of creation". +/// When you call [beginCell], you get a `builder`, populate it with binary data or structures, +/// and after [builder.endCell], you get a `cell`. +type builder = builtin + +/// `continuation` is an "executable cell" representing executable TVM bytecode. +/// They are used to manage execution flow in TVM programs and serve as +/// the basis for function calls, exception handling, and control flow operations. +type continuation = builtin + +/// `tuple` is a collection from 0 to 255 elements of any type. +/// You can push, pop, access individual elements as `someTuple.0`. +/// A tuple occupies one stack slot regardless of its size. +type tuple = builtin + +/// `address` represents an internal/external/none address. +/// Most likely, you'll use it for internal addresses — "an address of a smart contract". +/// It's `slice` under the hood. `someAddress as slice` is also possible. +/// [See docs](https://docs.ton.org/learn/overviews/addresses#address-of-smart-contract) +type address = builtin + +/// `never` is a special type that represents computations that never complete normally. +/// A function that always throws an exception returns `never`, and the compiler knows +/// that any code after it is unreachable. +type never = builtin + +/// `int8`, `int32`, `int222`, etc. is "a fixed-width signed integer with N bits", N <= 257. +/// Note: it's still `int` at runtime, you can assign "100500" to "int8": +/// overflow will happen at serialization to a cell/builder, NOT at assignment. +type intN = builtin + +/// `uint32`, `uint64`, `uint111`, etc. is "a fixed-width unsigned integer with N bits", N <= 256. +/// Note: it's still `int` at runtime, you can assign "100500" to "uint8": +/// overflow will happen at serialization to a cell/builder, NOT at assignment. +type uintN = builtin + +/// `coins` is a special primitive representing "nanotoncoins". One TON = 10^9 nanotoncoins. +/// You can create coins with `ton()` function: `ton("0.05")` (actually, `int` 50000000 at runtime). +/// Arithmetic operations on `coins` degrade to 257-bit `int` type. +type coins = builtin + +/// `varint16` is `int` at runtime, but serialized as "variadic signed int", -2^119 <= X < 2^119. +type varint16 = builtin + +/// `varuint16` is `int` at runtime, but serialized as "variadic unsigned int", 0 <= X < 2^120. +type varuint16 = builtin + +/// `varint32` is `int` at runtime, but serialized as "variadic signed int", -2^247 <= X < 2^247. +type varint32 = builtin + +/// `varuint32` is `int` at runtime, but serialized as "variadic unsigned int", 0 <= X < 2^248. +type varuint32 = builtin + +/// `bits256`, `bits111`, etc. is "a fixed-width slice with N bits and 0 refs", N <= 1023. +/// Note: use `as` operator to convert `slice` to `bitsN`: `someSlice as bits256` +/// (manually writing `as` enforces you to think that this conversion is correct). +/// Note: similar to `intN`, you can assign an invalid slice to `bitsN`, +/// an error will be fired at serialization with [T.toCell] and similar, NOT at assignment. +type bitsN = builtin + +/// `bytes8`, `bytes99`, etc. is a convenient alias for `bits(N*8)` +type bytesN = builtin + +/// `map` is "a map from a key K to a value V". +/// Internally, it's an "optional cell": an empty map is `null`, a non-empty points to a root cell. +/// Restrictions for K and V types: +/// - a key must be fixed-width; valid: `int32`, `uint64`, `address`, `bits256`, `Point`; invalid: `int`, `coins` +/// - a value must be serializable; valid: `int32`, `coins`, `AnyStruct`, `Cell`; invalid: `int`, `builder` +struct map { + private tvmDict: dict +} + +/// `dict` is a low-level TVM dictionary. +/// Think of it as "a map with unknown keys and unknown values". +/// Prefer using `map`, not `dict`. type dict = cell? +/// `void` is the unit type representing the absence of a meaningful value. +/// It's similar to both `void` and `unit` in other languages. +/// Note: a function without return type means "auto infer", NOT "void". +type void = builtin + +/// `Cell` represents a typed cell reference (as opposed to untyped `cell`). +/// ``` +/// struct ExtraData { ... } +/// +/// struct MyStorage { +/// ... +/// extra: Cell // TL-B `^ExtraData` +/// optional: Cell? // TL-B `(Maybe ^ExtraData)` +/// code: cell // TL-B `^Cell` +/// data: cell? // TL-B `(Maybe ^Cell)` +/// } +/// ``` +/// Note, that `st = MyStorage.fromSlice(s)` does NOT deep-load any refs; `st.extra` is `Cell`, not `T`; +/// you should manually call `st.extra.load()` to get `T` (ExtraData in this example). +struct Cell { + private readonly tvmCell: cell +} + + /** Tuple manipulation primitives. Elements of a tuple can be of arbitrary type. @@ -32,14 +150,14 @@ fun tuple.push(mutate self, value: T): void fun tuple.first(self): T asm "FIRST" -/// Returns the [`index`]-th element of a tuple. +/// Returns the `i`-th element of a tuple. /// `t.i` is actually the same as `t.get(i)` @pure fun tuple.get(self, index: int): T builtin -/// Sets the [`index`]-th element of a tuple to a specified value -/// (element with this index must already exist, a new element isn't created). +/// Sets the `i`-th element of a tuple to a specified value +/// (an element with this index must already exist, a new element is not created). /// `t.i = value` is actually the same as `t.set(value, i)` @pure fun tuple.set(mutate self, value: T, index: int): void @@ -67,7 +185,7 @@ fun tuple.pop(mutate self): T /// Converts a constant floating-point string to nanotoncoins. /// Example: `ton("0.05")` is equal to 50000000. -/// Note, that `ton()` requires a constant string; `ton(some_var)` is an error +/// Note, that `ton()` requires a constant string; `ton(some_var)` is an error. @pure fun ton(floatString: slice): coins builtin @@ -98,33 +216,33 @@ fun abs(x: int): int fun sign(x: int): int asm "SGN" -/// Computes the quotient and remainder of [x] / [y]. Example: divMod(112,3) = (37,1) +/// Computes the quotient and remainder of x / y. Example: divMod(112,3) = (37,1) @pure fun divMod(x: int, y: int): (int, int) asm "DIVMOD" -/// Computes the remainder and quotient of [x] / [y]. Example: modDiv(112,3) = (1,37) +/// Computes the remainder and quotient of x / y. Example: modDiv(112,3) = (1,37) @pure fun modDiv(x: int, y: int): (int, int) asm(-> 1 0) "DIVMOD" -/// Computes multiple-then-divide: floor([x] * [y] / [z]). +/// Computes multiple-then-divide: floor(x * y / z). /// The intermediate result is stored in a 513-bit integer to prevent precision loss. @pure fun mulDivFloor(x: int, y: int, z: int): int builtin -/// Similar to `mulDivFloor`, but rounds the result: round([x] * [y] / [z]). +/// Similar to [mulDivFloor], but rounds the result: round(x * y / z). @pure fun mulDivRound(x: int, y: int, z: int): int builtin -/// Similar to `mulDivFloor`, but ceils the result: ceil([x] * [y] / [z]). +/// Similar to [mulDivFloor], but ceils the result: ceil(x * y / z). @pure fun mulDivCeil(x: int, y: int, z: int): int builtin -/// Computes the quotient and remainder of ([x] * [y] / [z]). Example: mulDivMod(112,3,10) = (33,6) +/// Computes the quotient and remainder of (x * y / z). Example: mulDivMod(112,3,10) = (33,6) @pure fun mulDivMod(x: int, y: int, z: int): (int, int) builtin @@ -145,16 +263,13 @@ fun contract.getAddress(): address asm "MYADDR" /// Returns the balance (in nanotoncoins) of the smart contract at the start of Computation Phase. -/// Note that RAW primitives such as [sendMessage] do not update this field. @pure fun contract.getOriginalBalance(): coins asm "BALANCE" "FIRST" -/// Same as [contract.getOriginalBalance], but returns a tuple: -/// `int` — balance in nanotoncoins; -/// `dict` — a dictionary with 32-bit keys representing the balance of "extra currencies". +/// Same as [contract.getOriginalBalance], but returns a tuple: balance in nanotoncoins and extra currencies. @pure -fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, dict] +fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, ExtraCurrenciesMap] asm "BALANCE" /// Returns the persistent contract storage cell. It can be parsed or modified with slice and builder primitives later. @@ -162,17 +277,16 @@ fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, dict] fun contract.getData(): cell asm "c4 PUSH" -/// Sets `cell` [c] as persistent contract data. You can update persistent contract storage with this primitive. +/// Sets the persistent contract storage. fun contract.setData(c: cell): void asm "c4 POP" -/// Retrieves code of smart-contract from c7 +/// Retrieves code of smart-contract from c7. @pure fun contract.getCode(): cell asm "MYCODE" -/// Creates an output action that would change this smart contract code to that given by cell [newCode]. -/// Notice that this change will take effect only after the successful termination of the current run of the smart contract. +/// Creates an output action that would change this smart contract code after successful termination of the current run. fun contract.setCodePostponed(newCode: cell): void asm "SETCODE" @@ -208,8 +322,8 @@ fun blockchain.currentBlockLogicalTime(): int fun blockchain.configParam(x: int): cell? asm "CONFIGOPTPARAM" -/// Commits the current state of registers `c4` (“persistent data”) and `c5` (“actions”) -/// so that the current execution is considered “successful” with the saved values even if an exception +/// Commits current state of registers `c4` (persistent data) and `c5` (actions) +/// so that the current execution is considered "successful" with the saved values even if an exception /// in Computation Phase is thrown later. fun commitContractDataAndActions(): void asm "COMMIT" @@ -228,7 +342,7 @@ struct PackOptions { /// set this option to true to disable runtime checks; /// note: `int32` and other are always validated for overflow without any extra gas, /// so this flag controls only rarely used `bitsN` type - skipBitsNValidation: bool = false, + skipBitsNValidation: bool = false } /// UnpackOptions allows you to control behavior of `MyStruct.fromCell(c)` and similar functions. @@ -239,16 +353,15 @@ struct UnpackOptions { /// note: setting this to false does not decrease gas (DROP from a stack and ENDS cost the same); /// note: this option controls [T.fromCell] and [T.fromSlice], but is ignored by [slice.loadAny]; /// note: `lazy` ignores this option, because it reads fields on demand or even skips them - assertEndAfterReading: bool = true, + assertEndAfterReading: bool = true /// this excNo is thrown if a prefix doesn't match, e.g. for `struct (0x01) A` given input "88..."; /// similarly, for a union type, this is thrown when none of the opcodes match; /// note: `lazy` ignores this option if you have `else` in `match` (you write custom logic there) - throwIfOpcodeDoesNotMatch: int = 63, + throwIfOpcodeDoesNotMatch: int = 63 } /// Convert anything to a cell (most likely, you'll call it for structures). -/// Example: /// ``` /// var st: MyStorage = { ... }; /// contract.setData(st.toCell()); @@ -260,7 +373,6 @@ fun T.toCell(self, options: PackOptions = {}): Cell builtin /// Parse anything from a cell (most likely, you'll call it for structures). -/// Example: /// ``` /// var st = MyStorage.fromCell(contract.getData()); /// ``` @@ -271,11 +383,10 @@ fun T.fromCell(packedCell: cell, options: UnpackOptions = {}): T builtin /// Parse anything from a slice (most likely, you'll call it for structures). -/// Example: /// ``` /// var msg = CounterIncrement.fromSlice(cs); /// ``` -/// All fields are read from a slice immediately. +/// All fields are read from a slice immediately unless `lazy`. /// If a slice is corrupted, an exception is thrown (most likely, excode 9 "cell underflow"). /// Note, that a passed slice is NOT mutated, its internal pointer is NOT shifted. /// If you need to mutate it, like `cs.loadInt()`, consider calling `cs.loadAny()`. @@ -285,12 +396,11 @@ fun T.fromSlice(rawSlice: slice, options: UnpackOptions = {}): T /// Parse anything from a slice, shifting its internal pointer. /// Similar to `slice.loadUint()` and others, but allows loading structures. -/// Example: /// ``` /// var st: MyStorage = cs.loadAny(); // or cs.loadAny() /// ``` /// Similar to `MyStorage.fromSlice(cs)`, but called as a slice method and mutates the slice. -/// Note: [options.assertEndAfterReading] is ignored by this function, because it's actually intended +/// Note: `options.assertEndAfterReading` is ignored by this function, because it's actually intended /// to read data from the middle. @pure fun slice.loadAny(mutate self, options: UnpackOptions = {}): T @@ -298,7 +408,6 @@ fun slice.loadAny(mutate self, options: UnpackOptions = {}): T /// Skip anything in a slice, shifting its internal pointer. /// Similar to `slice.skipBits()` and others, but allows skipping structures. -/// Example: /// ``` /// struct TwoInts { a: int32; b: int32; } /// cs.skipAny(); // skips 64 bits @@ -309,7 +418,6 @@ fun slice.skipAny(mutate self, options: UnpackOptions = {}): self /// Store anything to a builder. /// Similar to `builder.storeUint()` and others, but allows storing structures. -/// Example: /// ``` /// var b = beginCell().storeUint(32).storeAny(msgBody).endCell(); /// ``` @@ -332,7 +440,6 @@ fun T.getDeclaredPackPrefixLen(): int /// Forces an object created by `lazy` to load fully. Returns the remaining slice (having read all fields). /// Since `options.assertEndAfterReading` is ignored by `lazy` (fields are loaded on demand), /// this method can help you overcome this, if you really need to check input consistency. -/// Example: /// ``` /// val msg = lazy CounterMessage.fromSlice(s); /// match (msg) { // it's a lazy match, without creating a union on the stack @@ -349,29 +456,9 @@ fun T.getDeclaredPackPrefixLen(): int fun T.forceLoadLazyObject(self): slice builtin -/// Cell represents a typed cell reference (as opposed to untyped `cell`). -/// Example: -/// ``` -/// struct ExtraData { ... } -/// -/// struct MyStorage { -/// ... -/// extra: Cell; // TL-B `^ExtraData` -/// optional: Cell?; // TL-B `(Maybe ^ExtraData)` -/// code: cell; // TL-B `^Cell` -/// data: cell?; // TL-B `(Maybe ^Cell)` -/// } -/// ``` -/// Note, that `st = MyStorage.fromSlice(s)` does NOT deep-load any refs; `st.extra` is `Cell`, not `T`; -/// you should manually call `st.extra.load()` to get T (ExtraData in this example). -struct Cell { - tvmCell: cell -} - /// Parse data from already loaded cell reference. -/// Example: /// ``` -/// struct MyStorage { ... extra: Cell; } +/// struct MyStorage { ... extra: Cell } /// /// var st = MyStorage.fromCell(contract.getData()); /// // st.extra is cell; if we need to unpack it, we do @@ -391,8 +478,7 @@ fun Cell.beginParse(self): slice fun Cell.hash(self): uint256 asm "HASHCU" -/// RemainingBitsAndRefs is a special built-in type to get "all the rest" slice tail on reading. -/// Example: +/// `RemainingBitsAndRefs` is a special built-in type to get "all the rest" slice tail on reading. /// ``` /// struct JettonMessage { /// ... some fields @@ -403,13 +489,13 @@ fun Cell.hash(self): uint256 type RemainingBitsAndRefs = slice /// Creates a cell with zero bits and references. -/// Equivalent to `beginCell().endCell()` but cheaper. +/// Equivalent to `beginCell().endCell()`. @pure fun createEmptyCell(): cell asm " PUSHREF" /// Creates a slice with zero remaining bits and references. -/// Equivalent to `beginCell().endCell().beginParse()` but cheaper. +/// Equivalent to `beginCell().endCell().beginParse()`. @pure fun createEmptySlice(): slice asm "x{} PUSHSLICE" @@ -420,76 +506,84 @@ fun createEmptySlice(): slice */ /// Compile-time function that calculates crc32 of a constant string. -/// Example: `const op = stringCrc32("some_str")` = 4013618352 = 0xEF3AF4B0 -/// Note: stringCrc32(slice_var) does not work! It accepts a constant string and works at compile-time. +/// ``` +/// const op = stringCrc32("some_str") // = 4013618352 = 0xEF3AF4B0 +/// ``` +/// Note: `stringCrc32(slice_var)` does NOT work! It accepts a constant string and works at compile-time. @pure fun stringCrc32(constString: slice): int builtin /// Compile-time function that calculates crc16 (XMODEM) of a constant string. -/// Example: `const op = stringCrc16("some_str")` = 53407 = 0xD09F -/// Note: stringCrc16(slice_var) does not work! It accepts a constant string and works at compile-time. +/// ``` +/// const op = stringCrc16("some_str") // = 53407 = 0xD09F +/// ``` +/// Note: `stringCrc16(slice_var)` does NOT work! It accepts a constant string and works at compile-time. @pure fun stringCrc16(constString: slice): int builtin /// Compile-time function that calculates sha256 of a constant string and returns 256-bit integer. -/// Example: `const hash = stringSha256("some_crypto_key")` -/// Note: it's a compile-time function, `stringSha256(slice_var)` does not work. -/// Use `sliceBitsHash` or `sliceHash` (declared below) to hash a slice without/with its refs at runtime. +/// ``` +/// const hash = stringSha256("some_crypto_key") +/// ``` +/// Note: it's a compile-time function, `stringSha256(slice_var)` does NOT work! +/// Use [slice.bitsHash] or [slice.hash] (declared below) to hash a slice without/with its refs at runtime. @pure fun stringSha256(constString: slice): int builtin /// Compile-time function that calculates sha256 of a constant string and takes the first 32 bits. -/// Example: `const minihash = stringSha256_32("some_crypto_key")` -/// Note: stringSha256_32(slice_var) does not work! It accepts a constant string and works at compile-time. +/// ``` +/// const minihash = stringSha256_32("some_crypto_key") +/// ``` +/// Note: `stringSha256_32(slice_var)` does NOT work! It accepts a constant string and works at compile-time. @pure fun stringSha256_32(constString: slice): int builtin /// Compile-time function that takes N-chars ascii string and interprets it as a number in base 256. -/// Example: `const value = stringToBase256("AB")` = 16706 (65*256 + 66) -/// Note: stringToBase256(slice_var) does not work! It accepts a constant string and works at compile-time. +/// ``` +/// const value = stringToBase256("AB") // = 16706 (65*256 + 66) +/// ``` +/// Note: `stringToBase256(slice_var)` does NOT work! It accepts a constant string and works at compile-time. @pure fun stringToBase256(constString: slice): int builtin -/// Computes the representation hash of a `cell` and returns it as a 256-bit unsigned integer `x`. +/// Computes the representation hash of a cell and returns it as a 256-bit unsigned integer. /// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. @pure fun cell.hash(self): uint256 asm "HASHCU" -/// Computes the hash of a `slice` and returns it as a 256-bit unsigned integer `x`. -/// The result is the same as if an ordinary cell containing only data and references from `s` had been created -/// and its hash computed by [cell.hash]. +/// Computes the hash of a slice and returns it as a 256-bit unsigned integer. +/// The result is the same as [cell.hash] for a cell containing data and references from this slice. @pure fun slice.hash(self): uint256 asm "HASHSU" -/// Computes sha256 of the data bits of a `slice`. If the bit length of `s` is not divisible by eight, -/// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. +/// Computes sha256 of the data bits of a slice (without refs). +/// If the bit length is not divisible by eight, throws a cell underflow exception. @pure fun slice.bitsHash(self): uint256 asm "SHA256U" /// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) -/// using [publicKey] (also represented by a 256-bit unsigned integer). +/// using `publicKey` (also represented by a 256-bit unsigned integer). /// The signature must contain at least 512 data bits; only the first 512 bits are used. -/// The result is `−1` if the signature is valid, `0` otherwise. /// Note that `CHKSIGNU` creates a 256-bit slice with the hash and calls `CHKSIGNS`. -/// That is, if [hash] is computed as the hash of some data, these data are hashed twice, +/// That is, if parameter `hash` is computed as the hash of some data, these data are hashed twice, /// the second hashing occurring inside `CHKSIGNS`. @pure fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool asm "CHKSIGNU" -/// Checks whether [signature] is a valid Ed25519-signature of the data portion of `slice data` using `publicKey`, +/// Checks whether `signature` is a valid Ed25519-signature of the data portion of slice `data` using `publicKey`, /// similarly to [isSignatureValid]. -/// If the bit length of [data] is not divisible by eight, throws a cell underflow exception. +/// If the bit length of `data` is not divisible by eight, throws a cell underflow exception. /// The verification of Ed25519 signatures is the standard one, -/// with sha256 used to reduce [data] to the 256-bit number that is actually signed. +/// with sha256 used to reduce `data` to the 256-bit number that is actually signed. @pure fun isSliceSignatureValid(data: slice, signature: slice, publicKey: int): bool asm "CHKSIGNS" @@ -535,20 +629,20 @@ fun random.initialize(): void /// Returns `(x, y, z, -1)` or `(null, null, null, 0)`. /// Recursively computes the count of distinct cells `x`, data bits `y`, and cell references `z` -/// in the DAG rooted at `cell` [c], effectively returning the total storage used by this DAG taking into account +/// in a tree of cells, effectively returning the total storage used by this tree taking into account /// the identification of equal cells. -/// The values of `x`, `y`, and `z` are computed by a depth-first traversal of this DAG, +/// The values of `x`, `y`, and `z` are computed by a depth-first traversal of this tree, /// with a hash table of visited cell hashes used to prevent visits of already-visited cells. -/// The total count of visited cells `x` cannot exceed non-negative [maxCells]; +/// The total count of visited cells `x` cannot exceed non-negative `maxCells`; /// otherwise the computation is aborted before visiting the `(maxCells + 1)`-st cell and -/// a zero flag is returned to indicate failure. If [c] is `null`, returns `x = y = z = 0`. +/// a zero flag is returned to indicate failure. @pure fun cell.calculateSize(self, maxCells: int): (int, int, int, bool) asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT" -/// Similar to [cell.calculateSize], but accepting a `slice` [s] instead of a `cell`. -/// The returned value of `x` does not take into account the cell that contains the `slice` [s] itself; -/// however, the data bits and the cell references of [s] are accounted for in `y` and `z`. +/// Similar to [cell.calculateSize], but accepting a slice instead of a cell. +/// The returned value of `x` does not take into account the cell that contains the slice itself; +/// however, its data bits and cell references are accounted for in `y` and `z`. @pure fun slice.calculateSize(self, maxCells: int): (int, int, int, bool) asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT" @@ -557,35 +651,30 @@ fun slice.calculateSize(self, maxCells: int): (int, int, int, bool) fun cell.calculateSizeStrict(self, maxCells: int): (int, int, int) asm "CDATASIZE" -/// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. +/// A non-quiet version of [slice.calculateSize] that throws a cell overflow exception (`8`) on failure. fun slice.calculateSizeStrict(self, maxCells: int): (int, int, int) asm "SDATASIZE" -/// Returns the depth of a `cell`. -/// If [c] has no references, then return `0`; -/// otherwise the returned value is one plus the maximum of depths of cells referred to from [c]. -/// If [c] is a `null` instead of a cell, returns zero. +/// Returns the depth of a cell: 0 if no references, otherwise 1 + maximum of depths of all references. +/// When called for `null`, returns 0. @pure fun cell?.depth(self): int asm "CDEPTH" -/// Returns the depth of a `slice`. -/// If [s] has no references, then returns `0`; -/// otherwise the returned value is one plus the maximum of depths of cells referred to from [s]. +/// Returns the depth of a cell: 0 if no references, otherwise 1 + maximum of depths of all references. @pure fun slice.depth(self): int asm "SDEPTH" -/// Returns the depth of a `builder`. -/// If no cell references are stored in [b], then returns 0; -/// otherwise the returned value is one plus the maximum of depths of cells referred to from [b]. +/// Returns the depth of a builder: 0 if no references, otherwise 1 + maximum of depths of all references. @pure fun builder.depth(self): int asm "BDEPTH" -/// Returns the number of stack slots anyVariable occupies (works at compile-time). -/// Example: sizeof(nullableInt) = 1, because `int?` is 1 TVM slot holding either NULL or a value. -/// Example: sizeof(somePoint) = 2 for `struct Point { x:int, y: int }`: two fields one slot per each. +/// Returns the number of stack slots `anyVariable` occupies (works at compile-time). +/// - sizeof(nullableInt) = 1, because `int?` is 1 TVM slot holding either NULL or a value. +/// - sizeof(somePoint) = 2 for `struct Point { x:int, y:int }`: two fields one slot per each. +/// /// Useful for debugging or when preparing stack contents for RUNVM. @pure fun sizeof(anyVariable: T): int @@ -601,11 +690,11 @@ fun sizeof(anyVariable: T): int /// Example: `debug.print(v)` and other methods. struct debug -/// Dump a variable [x] to the debug log. +/// Dump a variable to the debug log. fun debug.print(x: T): void builtin -/// Dump a string [x] to the debug log. +/// Dump a string to the debug log. fun debug.printString(x: T): void builtin @@ -614,6 +703,268 @@ fun debug.dumpStack(): void builtin +/* + Implementation of `map` and methods over it. + + Note: maps in TVM are called "dictionaries". They are stored as nested cells + (each value in a separate cell, probably with intermediate cells-nodes). + Constructing even small dicts is gas-expensive, because cell creation costs a lot. + + `map` is a "human-readable interface" over low-level dictionaries. The compiler, + knowing types of K and V, effectively generates asm instructions for storing/loading values. + But still, of course, it's a tree of cells, it's just a TVM dictionary. + */ + +/// Returns an empty typed map. It's essentially "PUSHNULL", since TVM NULL represents an empty map. +/// ``` +/// var m: map = createEmptyMap(); +/// ``` +@pure +fun createEmptyMap(): map + builtin + +/// Converts a low-level TVM dictionary to a typed map. +/// Actually, does nothing: accepts an "optional cell" and returns the same "optional cell", +/// so if you specify key/value types incorrectly, it will fail later, at `map.get` and similar. +@pure +fun createMapFromLowLevelDict(d: dict): map + builtin + +/// Converts a high-level map to a low-level TVM dictionary. +/// Actually, does nothing: returns the same "optional cell". +@pure +fun map.toLowLevelDict(self): dict + asm "NOP" + +/// Checks whether a map is empty (whether a cell is null). +/// +/// Note: a check `m == null` will not work, use `m.isEmpty()`. +@pure +fun map.isEmpty(self): bool + asm "DICTEMPTY" + +/// Checks whether a key exists in a map. +@pure +fun map.exists(self, key: K): bool + builtin + +/// Gets an element by key. If not found, does NOT throw, just returns isFound = false. +/// ``` +/// val r = m.get(123); +/// if (r.isFound) { +/// r.loadValue() +/// } +/// ``` +@pure +fun map.get(self, key: K): MapLookupResult + builtin + +/// Gets an element by key and throws if it doesn't exist. +@pure +fun map.mustGet(self, key: K, throwIfNotFound: int = 9): V + builtin + +/// Sets an element by key. +/// ``` +/// m.set(k, 3); +/// ``` +/// Since it returns `self`, calls may be chained. +@pure +fun map.set(mutate self, key: K, value: V): self + builtin + +/// Sets an element and returns the previous element at that key. If no previous, isFound = false. +/// ``` +/// val prev = m.setAndGetPrevious(k, 3); +/// if (prev.isFound) { +/// prev.loadValue() +/// } +/// ``` +@pure +fun map.setAndGetPrevious(mutate self, key: K, value: V): MapLookupResult + builtin + +/// Sets an element only if the key already exists. Returns whether an element was replaced. +@pure +fun map.replaceIfExists(mutate self, key: K, value: V): bool + builtin + +/// Sets an element only if the key already exists and returns the previous element at that key. +@pure +fun map.replaceAndGetPrevious(mutate self, key: K, value: V): MapLookupResult + builtin + +/// Sets an element only if the key does not exist. Returns whether an element was added. +@pure +fun map.addIfNotExists(mutate self, key: K, value: V): bool + builtin + +/// Sets an element only if the key does not exist. If exists, returns an old value. +@pure +fun map.addOrGetExisting(mutate self, key: K, value: V): MapLookupResult + builtin + +/// Delete an element at the key. Returns whether an element was deleted. +@pure +fun map.delete(mutate self, key: K): bool + builtin + +/// Delete an element at the key and returns the deleted element. If not exists, isFound = false. +/// ``` +/// val prev = m.deleteAndGetDeleted(k); +/// if (prev.isFound) { +/// prev.loadValue() +/// } +/// ``` +@pure +fun map.deleteAndGetDeleted(mutate self, key: K): MapLookupResult + builtin + +/// Finds the first (minimal) element in a map. If key are integers, it's the minimal integer. +/// If keys are addresses or complex structures (represented as slices), it's lexicographically smallest. +/// For an empty map, just returns isFound = false. +/// Useful for iterating over a map: +/// ``` +/// var r = m.findFirst(); +/// while (r.isFound) { +/// // ... use r.getKey() and r.loadValue() +/// r = m.iterateNext(r) +/// } +/// ``` +@pure +fun map.findFirst(self): MapEntry + builtin + +/// Finds the last (maximal) element in a map. If key are integers, it's the maximal integer. +/// If keys are addresses or complex structures (represented as slices), it's lexicographically largest. +/// For an empty map, just returns isFound = false. +/// Useful for iterating over a map: +/// ``` +/// var r = m.findLast(); +/// while (r.isFound) { +/// // ... use r.getKey() and r.loadValue() +/// r = m.iteratePrev(r) +/// } +/// ``` +@pure +fun map.findLast(self): MapEntry + builtin + +/// Finds an element with key > pivotKey. +/// Don't forget to check `isFound` before using `getKey()` and `loadValue()` of the result. +@pure +fun map.findKeyGreater(self, pivotKey: K): MapEntry + builtin + +/// Finds an element with key >= pivotKey. +/// Don't forget to check `isFound` before using `getKey()` and `loadValue()` of the result. +@pure +fun map.findKeyGreaterOrEqual(self, pivotKey: K): MapEntry + builtin + +/// Finds an element with key < pivotKey. +/// Don't forget to check `isFound` before using `getKey()` and `loadValue()` of the result. +@pure +fun map.findKeyLess(self, pivotKey: K): MapEntry + builtin + +/// Finds an element with key <= pivotKey. +/// Don't forget to check `isFound` before using `getKey()` and `loadValue()` of the result. +@pure +fun map.findKeyLessOrEqual(self, pivotKey: K): MapEntry + builtin + +/// Iterate over a map in ascending order. +/// ``` +/// // iterate for all keys >= 10 up to the end +/// var r = m.findKeyGreaterOrEqual(10); +/// while (r.isFound) { +/// // ... use r.getKey() and r.loadValue() +/// r = m.iterateNext(r) +/// } +/// ``` +@pure +fun map.iterateNext(self, current: MapEntry): MapEntry + builtin + +/// Iterate over a map in reverse order. +/// ``` +/// // iterate for all keys < 10 down lo lowest +/// var r = m.findKeyLess(10); +/// while (r.isFound) { +/// // ... use r.getKey() and r.loadValue() +/// r = m.iteratePrev(r) +/// } +/// ``` +@pure +fun map.iteratePrev(self, current: MapEntry): MapEntry + builtin + +/// `MapLookupResult` is a return value of `map.get`, `map.setAndGetPrevious`, and similar. +/// Instead of returning a nullable value (that you'd check on null before usage), +/// this struct is returned (and you check `isFound` before usage). +/// ``` +/// val r = m.get(key); +/// if (r.isFound) { +/// r.loadValue() // unpacks a returned slice +/// } +/// ``` +struct MapLookupResult { + private readonly rawSlice: slice? // holds encoded value, present if isFound + isFound: bool +} + +@pure +fun MapLookupResult.loadValue(self): TValue { + // it's assumed that you check for `isFound` before calling `loadValue()`; + // note that `assertEnd` is called to ensure no remaining data left besides TValue + return TValue.fromSlice(self.rawSlice!, { + assertEndAfterReading: true + }) +} + +@pure +fun MapLookupResult.loadValue(self): slice { + return self.rawSlice! +} + +/// `MapEntry` is a return value of `map.findFirst`, `map.iterateNext`, and similar. +/// You should check for `isFound` before calling `getKey()` and `loadValue()`. +/// ``` +/// var r = m.findFirst(); +/// while (r.isFound) { +/// // ... use r.getKey() and r.loadValue() +/// r = m.iterateNext(r) +/// } +/// ``` +struct MapEntry { + private readonly rawValue: slice? // holds encoded value, present if isFound + private readonly key: K + isFound: bool +} + +@pure +fun MapEntry.getKey(self): K { + // it's assumed that you check for `isFound` before calling `getKey()`; + // (otherwise, it will contain an incorrect stack slot with `null` value, not K) + return self.key +} + +@pure +fun MapEntry.loadValue(self): V { + // it's assumed that you check for `isFound` before calling `loadValue()`; + // note that `assertEnd` is inserted to ensure no remaining data left besides TValue + return V.fromSlice(self.rawValue!, { + assertEndAfterReading: true + }) +} + +@pure +fun MapEntry.loadValue(self): slice { + return self.rawValue! +} + + /** Slice primitives: parsing cells. When you _load_ some data, you mutate the slice (shifting an internal pointer on the stack). @@ -621,74 +972,74 @@ fun debug.dumpStack(): void */ /// Compile-time function that converts a constant hex-encoded string to N/2 bytes. -/// Example: `const v = stringHexToSlice("abcdef")` = slice with 3 bytes `[ 0xAB, 0xCD, 0xEF ]` -/// Note: stringHexToSlice(slice_var) does not work! It accepts a constant string and works at compile-time. +/// ``` +/// const v = stringHexToSlice("abcdef") // = slice with 3 bytes: 0xAB,0xCD,0xEF +/// ``` +/// Note: `stringHexToSlice(slice_var)` does NOT work! It accepts a constant string and works at compile-time. @pure fun stringHexToSlice(constStringBytesHex: slice): slice builtin -/// Converts a `cell` into a `slice`. Notice that [c] must be either an ordinary cell, -/// or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) -/// which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. +/// Converts a cell into a slice. @pure fun cell.beginParse(self): slice asm "CTOS" -/// Checks if slice is empty. If not, throws an exception with code 9. +/// Checks if a slice is empty. If not, throws an exception with code 9. fun slice.assertEnd(self): void asm "ENDS" -/// Loads the next reference from the slice. +/// Loads the next reference from a slice. @pure fun slice.loadRef(mutate self): cell asm( -> 1 0) "LDREF" -/// Preloads the next reference from the slice. +/// Preloads the next reference from a slice. @pure fun slice.preloadRef(self): cell asm "PLDREF" -/// Loads a signed [len]-bit integer from a slice. +/// Loads a signed len-bit integer from a slice. @pure fun slice.loadInt(mutate self, len: int): int builtin -/// Loads an unsigned [len]-bit integer from a slice. +/// Loads an unsigned len-bit integer from a slice. @pure fun slice.loadUint(mutate self, len: int): int builtin -/// Loads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice `s''`. +/// Loads the first `0 ≤ len ≤ 1023` bits from a slice. @pure fun slice.loadBits(mutate self, len: int): slice builtin -/// Preloads a signed [len]-bit integer from a slice. +/// Preloads a signed len-bit integer from a slice. @pure fun slice.preloadInt(self, len: int): int builtin -/// Preloads an unsigned [len]-bit integer from a slice. +/// Preloads an unsigned len-bit integer from a slice. @pure fun slice.preloadUint(self, len: int): int builtin -/// Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice. +/// Preloads the first `0 ≤ len ≤ 1023` bits from a slice. @pure fun slice.preloadBits(self, len: int): slice builtin -/// Loads serialized amount of Toncoins (any unsigned integer up to `2^120 - 1`). +/// Loads the serialized amount of nanotoncoins (any unsigned integer up to `2^120 - 1`). @pure fun slice.loadCoins(mutate self): coins asm( -> 1 0) "LDGRAMS" -/// Loads bool (-1 or 0) from a slice +/// Loads a boolean from a slice. @pure fun slice.loadBool(mutate self): bool asm( -> 1 0) "1 LDI" -/// Shifts a slice pointer to [len] bits forward, mutating the slice. +/// Shifts a slice pointer to len bits forward. @pure fun slice.skipBits(mutate self, len: int): self builtin @@ -715,7 +1066,6 @@ fun slice.getMiddleBits(self, offset: int, len: int): slice asm "SDSUBSTR" /// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. -/// Returns `null` if `nothing` constructor is used. @pure fun slice.loadDict(mutate self): dict asm( -> 1 0) "LDDICT" @@ -725,7 +1075,7 @@ fun slice.loadDict(mutate self): dict fun slice.preloadDict(self): dict asm "PLDDICT" -/// Loads a dictionary as [slice.loadDict], but returns only the remainder of the slice. +/// Skip a dictionary from a slice. @pure fun slice.skipDict(mutate self): self asm "SKIPDICT" @@ -741,11 +1091,12 @@ fun slice.loadMaybeRef(mutate self): cell? fun slice.preloadMaybeRef(self): cell? asm "PLDOPTREF" -/// Loads (Maybe ^Cell), but returns only the remainder of the slice. +/// Skips (Maybe ^Cell) in a slice. @pure fun slice.skipMaybeRef(mutate self): self asm "SKIPOPTREF" + /** Builder primitives: constructing cells. When you _store_ some data, you mutate the builder (shifting an internal pointer on the stack). @@ -758,7 +1109,7 @@ fun slice.skipMaybeRef(mutate self): self fun beginCell(): builder asm "NEWC" -/// Converts a builder into an ordinary `cell`. +/// Converts a builder into an ordinary cell. @pure fun builder.endCell(self): cell asm "ENDC" @@ -768,12 +1119,12 @@ fun builder.endCell(self): cell fun builder.storeRef(mutate self, c: cell): self asm(c self) "STREF" -/// Stores a signed [len]-bit integer into a builder (`0 ≤ len ≤ 257`). +/// Stores a signed len-bit integer into a builder (`0 ≤ len ≤ 257`). @pure fun builder.storeInt(mutate self, x: int, len: int): self builtin -/// Stores an unsigned [len]-bit integer into a builder (`0 ≤ len ≤ 256`). +/// Stores an unsigned len-bit integer into a builder (`0 ≤ len ≤ 256`). @pure fun builder.storeUint(mutate self, x: int, len: int): self builtin @@ -788,25 +1139,24 @@ fun builder.storeSlice(mutate self, s: slice): self fun builder.storeAddress(mutate self, addr: address): self asm(addr self) "STSLICE" -/// Stores amount of Toncoins into a builder. +/// Stores the amount of nanotoncoins into a builder. @pure fun builder.storeCoins(mutate self, x: coins): self builtin -/// Stores bool (-1 or 0) into a builder. -/// Attention: true value is `-1`, not 1! If you pass `1` here, TVM will throw an exception. +/// Stores a boolean into a builder. @pure fun builder.storeBool(mutate self, x: bool): self builtin -/// Stores dictionary (represented by TVM `cell` or `null`) into a builder. -/// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. +/// Stores a low-level TVM dictionary (optional cell) into a builder. +/// In other words, if `c` is null, stores `0`; if `c` is not null, stores `1` and a reference to `c`. @pure fun builder.storeDict(mutate self, c: dict): self asm(c self) "STDICT" /// Stores (Maybe ^Cell) into a builder. -/// In other words, if cell is `null`, store '0' bit; otherwise, store '1' and a ref to [c]. +/// In other words, if `c` is null, stores `0`; if `c` is not null, stores `1` and a reference to `c`. @pure fun builder.storeMaybeRef(mutate self, c: cell?): self asm(c self) "STOPTREF" @@ -816,7 +1166,7 @@ fun builder.storeMaybeRef(mutate self, c: cell?): self fun builder.storeBuilder(mutate self, from: builder): self asm(from self) "STB" -/// Stores a slice representing TL addr_none$00 (two `0` bits). +/// Stores a slice representing "none address" (TL addr_none$00 — two zero bits). @pure fun builder.storeAddressNone(mutate self): self asm "b{00} STSLICECONST" @@ -877,8 +1227,10 @@ fun builder.bitsCount(self): int */ /// Compile-time function that parses a valid contract address. -/// Example: address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") -/// Example: address("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") +/// ``` +/// address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") +/// address("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") +/// ``` /// Returns `address`, which can be stored in a builder, compared with `==`, etc. @pure fun address(stdAddress: slice): address @@ -889,9 +1241,8 @@ fun address(stdAddress: slice): address fun createAddressNone(): address asm "b{00} PUSHSLICE" -/// Returns if it's an empty address. -/// Don't confuse it with null! Empty address is a slice with two `0` bits. -/// In TL/B, it's addr_none$00. +/// Returns if it's an empty address (= "none address"). +/// Don't confuse it with null! An empty address is a slice "00" (two zero bits). @pure fun address.isNone(self): bool asm "b{00} SDBEGINSQ" "NIP" @@ -930,7 +1281,7 @@ fun address.getWorkchain(self): int8 fun address.bitsEqual(self, b: address): bool asm "SDEQ" -/// Loads from slice [s] a valid `MsgAddress` (none/internal/external). +/// Loads a valid address (internal/external/none). In TL/B, it's `MsgAddress`. @pure fun slice.loadAddress(mutate self): address asm( -> 1 0) "LDMSGADDR" @@ -954,7 +1305,7 @@ const RESERVE_MODE_NEGATE_AMOUNT = 8 const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16 /// Creates an output action which would reserve Toncoins on balance. -/// For [reserveMode] consider constants above. +/// For `reserveMode` consider constants above. fun reserveToncoinsOnBalance(nanoTonCoins: coins, reserveMode: int): void asm "RAWRESERVE" @@ -979,7 +1330,7 @@ const SEND_MODE_REGULAR = 0 const SEND_MODE_PAY_FEES_SEPARATELY = 1 /// +2 means that any errors arising while processing this message during the action phase should be ignored. const SEND_MODE_IGNORE_ERRORS = 2 -/// in the case of action fail - bounce transaction. No effect if SEND_MODE_IGNORE_ERRORS (+2) is used. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages +/// in the case of action fail - bounce transaction. No effect if SEND_MODE_IGNORE_ERRORS (+2) is used. [TVM UPGRADE 2023-07](https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages) const SEND_MODE_BOUNCE_ON_ACTION_FAIL = 16 /// mode = 32 means that the current account must be destroyed if its resulting balance is zero. const SEND_MODE_DESTROY = 32 @@ -987,31 +1338,33 @@ const SEND_MODE_DESTROY = 32 const SEND_MODE_CARRY_ALL_REMAINING_MESSAGE_VALUE = 64 /// mode = 128 is used for messages that are to carry all the remaining balance of the current smart contract (instead of the value originally indicated in the message). const SEND_MODE_CARRY_ALL_BALANCE = 128 -/// do not create an action, only estimate fee. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages +/// do not create an action, only estimate fee. [TVM UPGRADE 2023-07](https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages) const SEND_MODE_ESTIMATE_FEE_ONLY = 1024 /// Other modes affect the fee calculation as follows: /// +64 substitutes the entire balance of the incoming message as an outcoming value (slightly inaccurate, gas expenses that cannot be estimated before the computation is completed are not taken into account). /// +128 substitutes the value of the entire balance of the contract before the start of the computation phase (slightly inaccurate, since gas expenses that cannot be estimated before the completion of the computation phase are not taken into account). -type ExtraCurrenciesDict = dict +/// `ExtraCurrenciesMap` represents a dictionary of "extra currencies" other than TON coin. +/// They can be attached to a message (incoming and outgoing) and stored on contract's balance. +/// It's a dictionary `[currencyID => amount]` (amount is like `coins`, but encoded with a high precision). +type ExtraCurrenciesMap = map -/// ContractState is "code + data" of a contract. +/// `ContractState` is "code + data" of a contract. /// Used in outgoing messages (StateInit) to initialize a destination contract. struct ContractState { code: cell data: cell } -/// AddressShardingOptions provides settings to calculate an address in another shard. +/// `AddressShardingOptions` provides settings to calculate an address in another shard. /// Consider [createMessage] and [address.buildSameAddressInAnotherShard] for usage. struct AddressShardingOptions { fixedPrefixLength: uint5 // shard depth, formerly splitDepth closeTo: address } -/// AutoDeployAddress is a destination that initializes a receiver contract if it does not exist yet. +/// `AutoDeployAddress` is a destination that initializes a receiver contract if it does not exist yet. /// In order to do this, it contains StateInit — and calculates an address by a hash of StateInit. -/// Example: /// ``` /// createMessage({ /// dest: { @@ -1025,7 +1378,8 @@ struct AddressShardingOptions { /// ``` /// You just provide code+data, and the compiler automatically calculates the destination address, /// because in TON, the address of a contract, by definition, is a hash of its initial state. -/// You can also use it without sending a message. See [buildAddress] and [addressMatches]. +/// You can also use it without sending a message. +/// See [AutoDeployAddress.buildAddress] and [AutoDeployAddress.addressMatches]. struct AutoDeployAddress { workchain: int8 = BASECHAIN stateInit: ContractState | cell @@ -1062,7 +1416,7 @@ struct CreateMessageOptions { /// whether a message will bounce back on error bounce: bool /// message value: attached tons (or tons + extra currencies) - value: coins | (coins, ExtraCurrenciesDict) + value: coins | (coins, ExtraCurrenciesMap) /// destination is either a provided address, or is auto-calculated by stateInit dest: address // either just send a message to some address | builder // ... or a manually constructed builder with a valid address @@ -1074,16 +1428,16 @@ struct CreateMessageOptions { /// Creates a message (`OutMessage`) — a well-formatted message cell. /// Typically, you just send it. In advanced scenarios, you can estimate fees or even postpone sending. -/// Example: /// ``` /// val reply = createMessage({ /// bounce: false, /// value: ton("0.05"), /// dest: senderAddress, -/// body: RequestedInfo { ... } // note: no `toCell`! just pass an object +/// body: RequestedInfo { ... } // note: no toCell! just pass an object /// }); /// reply.send(SEND_MODE_REGULAR); /// ``` +/// /// Hint: don't call `body.toCell()`, pass `body` directly! /// (if body is small, it will be inlined without an expensive cell creation) /// (if body is large, the compiler will automatically wrap it into a cell) @@ -1093,14 +1447,14 @@ fun createMessage(options: CreateMessageOptions): OutMessage builtin -/// ExtOutLogBucket is a variant of a custom external address for emitting logs "to the outer world". +/// `ExtOutLogBucket` is a variant of a custom external address for emitting logs "to the outer world". /// It includes some "topic" (arbitrary number), that determines the format of the message body. /// For example, you emit "deposit event" (reserving topic "deposit" = 123): -/// > `dest: ExtOutLogBucket { topic: 123 }, body: DepositData { ... }` +/// ``` +/// dest: ExtOutLogBucket { topic: 123 }, +/// body: DepositData { ... } +/// ``` /// and external indexers can index your emitted logs by destination address without parsing body. -/// Currently, external messages are used only for emitting logs (for viewing them in indexers). -/// In the future, there might be -/// > 0x01 ExtOutOffchainContract { adnl: uint256 | bits256 } /// Serialization details: '01' (addr_extern) + 256 (len) + 0x00 (prefix) + 248 bits = 267 in total struct (0x00) ExtOutLogBucket { topic: uint248 | bits248 @@ -1119,7 +1473,6 @@ struct CreateExternalLogMessageOptions { /// Creates an external message (`OutMessage`) — a well-formatted message cell. /// Typically, you just send it. In advanced scenarios, you can estimate fees or even postpone sending. -/// Example: /// ``` /// val emitMsg = createExternalLogMessage({ /// dest: createAddressNone(), @@ -1135,18 +1488,17 @@ struct CreateExternalLogMessageOptions { fun createExternalLogMessage(options: CreateExternalLogMessageOptions): OutMessage builtin -/// UnsafeBodyNoRef is used to prevent default behavior: when message body is potentially large, +/// `UnsafeBodyNoRef` is used to prevent default behavior: when message body is potentially large, /// it's packed into a separate ref. Wrapping body with this struct tells the compiler: /// "inline message body in-place, I guarantee it will fit". -/// Example: /// ``` -/// struct ProbablyLarge { a: (coins, coins, coins, coins, coins) } // max 620 bits +/// struct ProbablyLarge { a: (coins, coins, coins, coins, coins) } /// /// val contents: ProbablyLarge = { ... }; // you are sure: values are small /// createMessage({ -/// // body: contents, // by default, it will be stored a ref (it's large) +/// // body: contents, // by default, will be a ref (max 620 bits) /// body: UnsafeBodyNoRef { -/// forceInline: contents, // but wrapping forces the compiler to inline it +/// forceInline: contents, // but it forces the compiler to inline it /// } /// ``` /// Another example: your body contains `builder` or `RemainingBitsAndRefs`: unpredictable size. @@ -1178,7 +1530,7 @@ fun OutMessage.hash(self): uint256 asm "HASHCU" -/// StateInit is a "canonical TL/B representation from block.tlb" of a contract initial state. +/// `StateInit` is a "canonical TL/B representation from block.tlb" of a contract initial state. /// But for everyday tasks, it's too complicated. It is not used in practice. /// To represent code+data, consider [ContractState]. /// To represent sharding (fixedPrefixLength, formerly splitDepth), consider [AutoDeployAddress]. @@ -1191,7 +1543,6 @@ struct StateInit { } /// Calculates a hash of StateInit if only code+data are set. -/// Example: /// ``` /// val addrHash = StateInit.calcHashCodeData(codeCell, dataCell); /// createMessage({ @@ -1251,12 +1602,16 @@ fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: /// Given an internal address A="aaaa...a" returns "bbaa...a" (D bits from address B, 256-D from A). +/// /// Example for fixedPrefixLength (shard depth) = 8: -/// | self (A) | aaaaaaaaaaa...aaa | -/// | closeTo (B) | 01010101bbb...bbb | shardPrefix = 01010101 (depth 8) -/// | result | 01010101aaa...aaa | address of A in same shard as B +/// ``` +/// self (A) | aaaaaaaaaaa...aaa | +/// closeTo (B) | 01010101bbb...bbb | shardPrefix = 01010101 (depth 8) +/// result | 01010101aaa...aaa | address of A in same shard as B +/// ``` /// More precisely, self (input) is 267 bits: '100' (std addr no anycast) + workchainA + "aaaa...a". /// The result is also 267 bits: '100' + workchainB + "bb" (D bits) + "aa...a" (256-D bits). +/// /// Note: returns `builder`, not `address`! Because builder is cheap, and you can send a message to it: /// ``` /// createMessage({ @@ -1300,10 +1655,9 @@ fun sendRawMessage(msg: cell, mode: int): void Receiving and handling messages. */ -/// InMessage is an input for `onInternalMessage` — when your contract accepts a non-bounced message. +/// `InMessage` is an input for `onInternalMessage` — when your contract accepts a non-bounced message. /// Internally, some data exist on the stack, some can be acquired with TVM instructions. /// The compiler replaces `in.someField` and gets a value correctly. -/// Example: /// ``` /// fun onInternalMessage(in: InMessage) { /// in.senderAddress // actually calls `INMSG_SRC` asm instruction @@ -1312,18 +1666,17 @@ fun sendRawMessage(msg: cell, mode: int): void struct InMessage { senderAddress: address // an internal address from which the message arrived valueCoins: coins // ton amount attached to an incoming message - valueExtra: dict // extra currencies attached to an incoming message + valueExtra: ExtraCurrenciesMap // extra currencies attached to an incoming message originalForwardFee: coins // fee that was paid by the sender createdLt: uint64 // logical time when a message was created createdAt: uint32 // unixtime when a message was created body: slice // message body, parse it with `lazy AllowedMsg.fromSlice(in.body)` } -/// InMessageBounced is an input for `onBouncedMessage`. +/// `InMessageBounced` is an input for `onBouncedMessage`. /// Very similar to a non-bounced input [InMessage]. /// Note, that `bouncedBody` is currently 256 bits: 0xFFFFFFFF + 224 bits from the originally sent message. /// Parse it with care! -/// Example: /// ``` /// fun onBouncedMessage(in: InMessageBounced) { /// in.bouncedBody.skipBouncedPrefix(); @@ -1332,7 +1685,7 @@ struct InMessage { struct InMessageBounced { senderAddress: address // an internal address from which the message was bounced valueCoins: coins // ton amount attached to a message - valueExtra: dict // extra currencies attached to a message + valueExtra: ExtraCurrenciesMap // extra currencies attached to a message originalForwardFee: coins // comission that the sender has payed to send this message createdLt: uint64 // logical time when a message was created (and bounced) createdAt: uint32 // unixtime when a message was created (and bounced) diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 916cefc6c..dd343508f 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 1.0 +tolk 1.1 /** Gas and payment related primitives. @@ -32,16 +32,16 @@ fun setGasLimitToMaximum(): void fun setGasLimit(limit: int): void asm "SETGASLIMIT" -/// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed [gasUsed] gas units. +/// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed `gasUsed` gas units. fun calculateGasFee(workchain: int8, gasUsed: int): coins asm(gasUsed workchain) "GETGASFEE" -/// Same as [calculateGasFee], but without flat price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) +/// Same as [calculateGasFee], but without flat price. [Docs about fees](https://docs.ton.org/develop/howto/fees-low-level) fun calculateGasFeeWithoutFlatPrice(workchain: int8, gasUsed: coins): coins asm(gasUsed workchain) "GETGASFEESIMPLE" -/// Calculates amount of nanotoncoins you should pay for storing a contract of provided size for [seconds]. -/// [bits] and [cells] represent contract state (code + data). +/// Calculates amount of nanotoncoins you should pay for storing a contract of provided size for `seconds`. +/// `bits` and `cells` represent contract state (code + data). fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): coins asm(cells bits seconds workchain) "GETSTORAGEFEE" @@ -49,7 +49,7 @@ fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): c fun calculateForwardFee(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEE" -/// Same as [calculateForwardFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) +/// Same as [calculateForwardFee], but without lump price. [Docs about fees](https://docs.ton.org/develop/howto/fees-low-level) fun calculateForwardFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEESIMPLE" diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index 212586b1d..85d44e1a0 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 1.0 +tolk 1.1 /** Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 41e091f0a..dee3ea4d7 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,9 +1,12 @@ // A part of standard library for Tolk -tolk 1.0 +tolk 1.1 /** - Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). - Currently, they have very low-level API very close to TVM internals. + Low-level API working with TVM dictionaries. + Not recommended to use, since it's very complicated and error-prone. + Use `map` instead, it's even more efficient then low-level dict API. + + Dictionaries are represented as `cell` data type. Most of functions are duplicated for three common cases: - iDict* - dicts with signed integer keys - uDict* - dicts with unsigned integer keys @@ -14,10 +17,12 @@ tolk 1.0 */ /// In @stdlib/common.tolk, there is a type alias: -/// `type dict = cell?` +/// ``` +/// type dict = cell? +/// ``` /// For clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. -/// Creates an empty dictionary, which is actually a null value. Equivalent to PUSHNULL +/// Creates an empty dictionary, which is actually a null value. Equivalent to `PUSHNULL` @pure fun createEmptyDict(): dict asm "NEWDICT" @@ -86,28 +91,28 @@ fun dict.uDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): boo @pure -fun dict.iDictGetRef(self, keyLen: int, key: int): (dict, bool) +fun dict.iDictGetRef(self, keyLen: int, key: int): (cell?, bool) asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT" @pure -fun dict.uDictGetRef(self, keyLen: int, key: int): (dict, bool) +fun dict.uDictGetRef(self, keyLen: int, key: int): (cell?, bool) asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT" @pure -fun dict.sDictGetRef(self, keyLen: int, key: slice): (dict, bool) +fun dict.sDictGetRef(self, keyLen: int, key: slice): (cell?, bool) asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT" @pure -fun dict.iDictGetRefOrNull(self, keyLen: int, key: int): dict +fun dict.iDictGetRefOrNull(self, keyLen: int, key: int): cell? asm(key self keyLen) "DICTIGETOPTREF" @pure -fun dict.uDictGetRefOrNull(self, keyLen: int, key: int): dict +fun dict.uDictGetRefOrNull(self, keyLen: int, key: int): cell? asm(key self keyLen) "DICTUGETOPTREF" @pure -fun dict.sDictGetRefOrNull(self, keyLen: int, key: slice): dict +fun dict.sDictGetRefOrNull(self, keyLen: int, key: slice): cell? asm(key self keyLen) "DICTGETOPTREF" @@ -138,11 +143,11 @@ fun dict.sDictSetAndGet(mutate self, keyLen: int, key: slice, value: slice): (sl @pure -fun dict.iDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict +fun dict.iDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): cell? asm(value key self keyLen) "DICTISETGETOPTREF" @pure -fun dict.uDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict +fun dict.uDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): cell? asm(value key self keyLen) "DICTUSETGETOPTREF" @@ -228,15 +233,15 @@ fun dict.sDictGetFirst(self, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2" @pure -fun dict.iDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) +fun dict.iDictGetFirstAsRef(self, keyLen: int): (int?, cell?, bool) asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2" @pure -fun dict.uDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) +fun dict.uDictGetFirstAsRef(self, keyLen: int): (int?, cell?, bool) asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2" @pure -fun dict.sDictGetFirstAsRef(self, keyLen: int): (slice?, dict, bool) +fun dict.sDictGetFirstAsRef(self, keyLen: int): (slice?, cell?, bool) asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2" @@ -253,15 +258,15 @@ fun dict.sDictGetLast(self, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2" @pure -fun dict.iDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) +fun dict.iDictGetLastAsRef(self, keyLen: int): (int?, cell?, bool) asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2" @pure -fun dict.uDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) +fun dict.uDictGetLastAsRef(self, keyLen: int): (int?, cell?, bool) asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2" @pure -fun dict.sDictGetLastAsRef(self, keyLen: int): (slice?, dict, bool) +fun dict.sDictGetLastAsRef(self, keyLen: int): (slice?, cell?, bool) asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2" @@ -273,6 +278,10 @@ fun dict.iDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) fun dict.uDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2" +@pure +fun dict.sDictGetNext(self, keyLen: int, pivot: slice): (slice?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTGETNEXT" "NULLSWAPIFNOT2" + @pure fun dict.iDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2" @@ -281,6 +290,10 @@ fun dict.iDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool fun dict.uDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2" +@pure +fun dict.sDictGetNextOrEqual(self, keyLen: int, pivot: slice): (slice?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTGETNEXTEQ" "NULLSWAPIFNOT2" + @pure fun dict.iDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) @@ -290,6 +303,10 @@ fun dict.iDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) fun dict.uDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2" +@pure +fun dict.sDictGetPrev(self, keyLen: int, pivot: slice): (slice?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTGETPREV" "NULLSWAPIFNOT2" + @pure fun dict.iDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2" @@ -298,9 +315,13 @@ fun dict.iDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool fun dict.uDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2" +@pure +fun dict.sDictGetPrevOrEqual(self, keyLen: int, pivot: slice): (slice?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTGETPREVEQ" "NULLSWAPIFNOT2" + /** - Prefix dictionary primitives. + Prefix dictionary primitives. */ @pure diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index d578199c7..2885dc312 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 1.0 +tolk 1.1 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. @@ -14,12 +14,12 @@ fun getTvmRegisterC3(): continuation fun setTvmRegisterC3(c: continuation): void asm "c3 POP" -/// Transforms a `slice` [s] into a simple ordinary continuation `c`, with `c.code = s` and an empty stack and savelist. +/// Transforms a slice into a simple ordinary continuation, with `c.code = s` and an empty stack and savelist. @pure fun transformSliceToContinuation(s: slice): continuation asm "BLESS" -/// Moves a variable or a value [x] to the top of the stack. +/// Moves a variable to the top of the stack. @pure fun T.stackMoveToTop(mutate self): void asm "NOP" diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 265c602d7..c067b88ff 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -61,9 +61,9 @@ td::Ref build_internal_message(td::RefInt256 amount, td::Refbit_size(false) + 7) >> 3); b.store_long_bool(len, 4) && b.store_int256_bool(*amount, len * 8, false); // grams:Grams - b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extre, ihr_fee, fwd_fee, created_lt, created_at, init + b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extra currencies, extra_flags, fwd_fee, created_lt, created_at, init // body:(Either X ^X) - if (b.remaining_bits() >= 1 + (*body).size() && b.remaining_refs() >= (*body).size_refs()) { + if (b.remaining_bits() >= 1 + body->size() && b.remaining_refs() >= body->size_refs()) { b.store_zeroes(1); b.append_cellslice(body); } else { @@ -162,7 +162,9 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod vm::load_cell_slice_ref(address), // myself:MsgAddressInt vm::StackEntry::maybe(config) // vm::StackEntry::maybe(td::Ref()) }; - if (args.config && args.config.value()->get_global_version() >= 4) { + + int global_version = args.config ? args.config.value()->get_global_version() : SUPPORTED_VERSION; + if (global_version >= 4) { tuple.push_back(vm::StackEntry::maybe(code)); // code:Cell tuple.push_back(block::CurrencyCollection::zero().as_vm_tuple()); // in_msg_value:[Integer (Maybe Cell)] tuple.push_back(td::zero_refint()); // storage_fees:Integer @@ -173,17 +175,18 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod // prev_key_block:BlockId ] : PrevBlocksInfo tuple.push_back(args.prev_blocks_info ? args.prev_blocks_info.value() : vm::StackEntry{}); // prev_block_info } - if (args.config && args.config.value()->get_global_version() >= 6) { - tuple.push_back(args.config.value()->get_unpacked_config_tuple(now)); // unpacked_config_tuple + if (global_version >= 6) { + tuple.push_back(args.config ? args.config.value()->get_unpacked_config_tuple(now) + : vm::StackEntry{}); // unpacked_config_tuple tuple.push_back(td::zero_refint()); // due_payment // precompiled_gas_usage:(Maybe Integer) td::optional precompiled; - if (code.not_null()) { + if (code.not_null() && args.config) { precompiled = args.config.value()->get_precompiled_contracts_config().get_contract(code->get_hash().bits()); } tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry()); } - if (args.config && args.config.value()->get_global_version() >= 11) { + if (global_version >= 11) { tuple.push_back(block::transaction::Transaction::prepare_in_msg_params_tuple(nullptr, {}, {})); } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); @@ -263,7 +266,7 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Refdump(os, 2); LOG(DEBUG) << "VM stack:\n" << os.str(); } - int global_version = config ? config->get_global_version() : 0; + int global_version = config ? config->get_global_version() : SUPPORTED_VERSION; vm::VmState vm{state.code, global_version, std::move(stack), gas, 1, state.data, log}; vm.set_c7(std::move(c7)); vm.set_chksig_always_succeed(ignore_chksig); diff --git a/crypto/test/test-smartcont.cpp b/crypto/test/test-smartcont.cpp index 7f512ceae..a27840721 100644 --- a/crypto/test/test-smartcont.cpp +++ b/crypto/test/test-smartcont.cpp @@ -547,8 +547,8 @@ bool operator<(const ton::MultisigWallet::Mask& a, const ton::MultisigWallet::Ma TEST(Smartcon, Multisig) { auto ms_lib = ton::MultisigWallet::create(); - int n = 100; - int k = 99; + int n = 50; + int k = 49; td::uint32 wallet_id = std::numeric_limits::max() - 3; std::vector keys; for (int i = 0; i < n; i++) { @@ -613,10 +613,10 @@ TEST(Smartcon, Multisig) { LOG(INFO) << "CODE: " << ans.code; LOG(INFO) << "GAS: " << ans.gas_used; } - for (int i = 0; i + 1 < 50; i++) { + for (int i = 0; i + 1 < 25; i++) { qb.sign(i, keys[i]); } - auto query = qb.create(49, keys[49]); + auto query = qb.create(24, keys[24]); CHECK(ms->get_n_k() == std::make_pair(n, k)); auto ans = ms.write().send_external_message(query, args()); @@ -629,10 +629,10 @@ TEST(Smartcon, Multisig) { { ton::MultisigWallet::QueryBuilder qb(wallet_id, query_id, vm::CellBuilder().finalize()); - for (int i = 50; i + 1 < 100; i++) { + for (int i = 25; i + 1 < 50; i++) { qb.sign(i, keys[i]); } - query = qb.create(99, keys[99]); + query = qb.create(49, keys[49]); } ans = ms.write().send_external_message(query, args()); diff --git a/crypto/vm/boc-compression.cpp b/crypto/vm/boc-compression.cpp new file mode 100644 index 000000000..d37e0380a --- /dev/null +++ b/crypto/vm/boc-compression.cpp @@ -0,0 +1,625 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "boc-compression.h" + +#include +#include +#include "vm/boc.h" +#include "vm/boc-writers.h" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "td/utils/Slice-decl.h" +#include "td/utils/lz4.h" + +namespace vm { + +td::Result boc_compress_baseline_lz4(const std::vector>& boc_roots) { + TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(boc_roots), 2)); + td::BufferSlice compressed = td::lz4_compress(data); + + // Add decompressed size at the beginning + td::BufferSlice compressed_with_size(compressed.size() + kDecompressedSizeBytes); + auto size_slice = td::BitSliceWrite(compressed_with_size.as_slice().ubegin(), kDecompressedSizeBytes * 8); + size_slice.bits().store_uint(data.size(), kDecompressedSizeBytes * 8); + memcpy(compressed_with_size.data() + kDecompressedSizeBytes, compressed.data(), compressed.size()); + + return compressed_with_size; +} + +td::Result>> boc_decompress_baseline_lz4(td::Slice compressed, int max_decompressed_size) { + // Check minimum input size for decompressed size header + if (compressed.size() < kDecompressedSizeBytes) { + return td::Status::Error("BOC decompression failed: input too small for header"); + } + + // Read decompressed size + constexpr size_t kSizeBits = kDecompressedSizeBytes * 8; + int decompressed_size = td::BitSlice(compressed.ubegin(), kSizeBits).bits().get_uint(kSizeBits); + compressed.remove_prefix(kDecompressedSizeBytes); + if (decompressed_size <= 0 || decompressed_size > max_decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid decompressed size"); + } + + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); + return roots; +} + +inline void append_uint(td::BitString& bs, unsigned val, unsigned n) { + bs.reserve_bitslice(n).bits().store_uint(val, n); +} + +inline td::Result read_uint(td::BitSlice& bs, int bits) { + // Check if there enough bits available + if (bs.size() < bits) { + return td::Status::Error("BOC decompression failed: not enough bits to read"); + } + unsigned result = bs.bits().get_uint(bits); + bs.advance(bits); + return result; +} + +td::Result boc_compress_improved_structure_lz4(const std::vector>& boc_roots) { + // Input validation + if (boc_roots.empty()) { + return td::Status::Error("No root cells were provided for serialization"); + } + for (const auto& root : boc_roots) { + if (root.is_null()) { + return td::Status::Error("Cannot serialize a null cell reference into a bag of cells"); + } + } + + // Initialize data structures for graph representation + td::HashMap cell_hashes; + std::vector> boc_graph; + std::vector refs_cnt; + std::vector cell_data; + std::vector cell_type; + std::vector prunned_branch_level; + std::vector root_indexes; + size_t total_size_estimate = 0; + + // Build graph representation using recursive lambda + const auto build_graph = [&](auto&& self, td::Ref cell) -> td::Result { + if (cell.is_null()) { + return td::Status::Error("Error while importing a cell during serialization: cell is null"); + } + + auto cell_hash = cell->get_hash(); + auto it = cell_hashes.find(cell_hash); + if (it != cell_hashes.end()) { + return it->second; + } + + size_t current_cell_id = boc_graph.size(); + cell_hashes.emplace(cell_hash, current_cell_id); + + bool is_special = false; + vm::CellSlice cell_slice = vm::load_cell_slice_special(cell, is_special); + if (!cell_slice.is_valid()) { + return td::Status::Error("Invalid loaded cell data"); + } + td::BitSlice cell_bitslice = cell_slice.as_bitslice(); + + // Initialize new cell in graph + boc_graph.emplace_back(); + refs_cnt.emplace_back(cell_slice.size_refs()); + cell_type.emplace_back(size_t(cell_slice.special_type())); + prunned_branch_level.push_back(0); + + DCHECK(cell_slice.size_refs() <= 4); + + // Process special cell of type PrunnedBranch + if (cell_slice.special_type() == vm::CellTraits::SpecialType::PrunnedBranch) { + DCHECK(cell_slice.size() >= 16); + cell_data.emplace_back(cell_bitslice.subslice(16, cell_bitslice.size() - 16)); + prunned_branch_level.back() = cell_slice.data()[1]; + } else { + cell_data.emplace_back(cell_bitslice); + } + total_size_estimate += cell_bitslice.size(); + + // Process cell references + for (int i = 0; i < cell_slice.size_refs(); ++i) { + TRY_RESULT(child_id, self(self, cell_slice.prefetch_ref(i))); + boc_graph[current_cell_id][i] = child_id; + } + + return current_cell_id; + }; + + // Build the graph starting from roots + for (auto root : boc_roots) { + TRY_RESULT(root_cell_id, build_graph(build_graph, root)); + root_indexes.push_back(root_cell_id); + } + + // Check graph properties + const size_t node_count = boc_graph.size(); + std::vector> reverse_graph(node_count); + size_t edge_count = 0; + + // Build reverse graph + for (int i = 0; i < node_count; ++i) { + for (size_t child_index = 0; child_index < refs_cnt[i]; ++child_index) { + size_t child = boc_graph[i][child_index]; + ++edge_count; + reverse_graph[child].push_back(i); + } + } + + // Process cell data sizes + std::vector is_data_small(node_count, 0); + for (int i = 0; i < node_count; ++i) { + if (cell_type[i] != 1) { + is_data_small[i] = cell_data[i].size() < 128; + } + } + + // Perform topological sort + std::vector topo_order, rank(node_count); + const auto topological_sort = [&]() -> td::Status { + std::vector> queue; + queue.reserve(node_count); + std::vector in_degree(node_count); + + // Calculate in-degrees and initialize queue + for (int i = 0; i < node_count; ++i) { + in_degree[i] = refs_cnt[i]; + if (in_degree[i] == 0) { + queue.emplace_back(cell_type[i] == 0, -int(cell_data[i].size()), -i); + } + } + + if (queue.empty()) { + return td::Status::Error("Cycle detected in cell references"); + } + + std::sort(queue.begin(), queue.end()); + + // Process queue + while (!queue.empty()) { + int node = -std::get<2>(queue.back()); + queue.pop_back(); + topo_order.push_back(node); + + for (int parent : reverse_graph[node]) { + if (--in_degree[parent] == 0) { + queue.emplace_back(0, 0, -parent); + } + } + } + + if (topo_order.size() != node_count) { + return td::Status::Error("Invalid graph structure"); + } + + std::reverse(topo_order.begin(), topo_order.end()); + return td::Status::OK(); + }; + + TRY_STATUS(topological_sort()); + + // Calculate index of vertices in topsort + for (int i = 0; i < node_count; ++i) { + rank[topo_order[i]] = i; + } + + // Build compressed representation + td::BitString result; + total_size_estimate += (node_count * 10 * 8); + result.reserve_bits(total_size_estimate); + + // Store roots information + append_uint(result, root_indexes.size(), 32); + for (int root_ind : root_indexes) { + append_uint(result, rank[root_ind], 32); + } + + // Store node count + append_uint(result, node_count, 32); + + // Store cell types and sizes + for (int i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + size_t currrent_cell_type = bool(cell_type[node]) + prunned_branch_level[node]; + append_uint(result, currrent_cell_type, 4); + append_uint(result, refs_cnt[node], 4); + + if (cell_type[node] != 1) { + if (is_data_small[node]) { + append_uint(result, 1, 1); + append_uint(result, cell_data[node].size(), 7); + } else { + append_uint(result, 0, 1); + append_uint(result, 1 + cell_data[node].size() / 8, 7); + } + } + } + + // Store edge information + auto edge_bits = result.reserve_bitslice(edge_count).bits(); + for (int i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + for (size_t child_index = 0; child_index < refs_cnt[node]; ++child_index) { + size_t child = boc_graph[node][child_index]; + edge_bits.store_uint(rank[child] == i + 1, 1); + ++edge_bits; + } + } + + // Store cell data + for (size_t node : topo_order) { + if (cell_type[node] != 1 && !is_data_small[node]) { + continue; + } + result.append(cell_data[node].subslice(0, cell_data[node].size() % 8)); + } + + // Store BOC graph with optimized encoding + for (size_t i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + if (node_count <= i + 3) + continue; + + for (int j = 0; j < refs_cnt[node]; ++j) { + if (rank[boc_graph[node][j]] <= i + 1) + continue; + + int delta = rank[boc_graph[node][j]] - i - 2; // Always >= 0 because of above check + size_t required_bits = 1 + (31 ^ td::count_leading_zeroes32(node_count - i - 3)); + + if (required_bits < 8 - (result.size() + 1) % 8 + 1) { + append_uint(result, delta, required_bits); + } else if (delta < (1 << (8 - (result.size() + 1) % 8))) { + size_t available_bits = 8 - (result.size() + 1) % 8; + append_uint(result, 1, 1); + append_uint(result, delta, available_bits); + } else { + append_uint(result, 0, 1); + append_uint(result, delta, required_bits); + } + } + } + + // Pad result to byte boundary + while (result.size() % 8) { + append_uint(result, 0, 1); + } + + // Store remaining cell data + for (size_t node : topo_order) { + if (cell_type[node] == 1 || is_data_small[node]) { + size_t prefix_size = cell_data[node].size() % 8; + result.append(cell_data[node].subslice(prefix_size, cell_data[node].size() - prefix_size)); + } else { + size_t data_size = cell_data[node].size() + 1; + size_t padding = (8 - data_size % 8) % 8; + + if (padding) { + append_uint(result, 0, padding); + } + append_uint(result, 1, 1); + result.append(cell_data[node]); + } + } + + // Final padding + while (result.size() % 8) { + append_uint(result, 0, 1); + } + + // Create final compressed buffer + td::BufferSlice serialized((const char*)result.bits().get_byte_ptr(), result.size() / 8); + + td::BufferSlice compressed = td::lz4_compress(serialized); + + // Add decompressed size at the beginning + td::BufferSlice compressed_with_size(compressed.size() + kDecompressedSizeBytes); + auto size_slice = td::BitSliceWrite(compressed_with_size.as_slice().ubegin(), kDecompressedSizeBytes * 8); + size_slice.bits().store_uint(serialized.size(), kDecompressedSizeBytes * 8); + memcpy(compressed_with_size.data() + kDecompressedSizeBytes, compressed.data(), compressed.size()); + + return compressed_with_size; +} + +td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, int max_decompressed_size) { + constexpr size_t kMaxCellDataLengthBits = 1024; + + // Check minimum input size for decompressed size header + if (compressed.size() < kDecompressedSizeBytes) { + return td::Status::Error("BOC decompression failed: input too small for header"); + } + + // Read decompressed size + constexpr size_t kSizeBits = kDecompressedSizeBytes * 8; + size_t decompressed_size = td::BitSlice(compressed.ubegin(), kSizeBits).bits().get_uint(kSizeBits); + compressed.remove_prefix(kDecompressedSizeBytes); + if (decompressed_size > max_decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid decompressed size"); + } + + // Decompress LZ4 data + TRY_RESULT(serialized, td::lz4_decompress(compressed, decompressed_size)); + + if (serialized.size() != decompressed_size) { + return td::Status::Error("BOC decompression failed: decompressed size mismatch"); + } + + // Initialize bit reader + td::BitSlice bit_reader(serialized.as_slice().ubegin(), serialized.as_slice().size() * 8); + size_t orig_size = bit_reader.size(); + + // Read root count + TRY_RESULT(root_count, read_uint(bit_reader, 32)); + // We assume that each cell should take at least 1 byte, even effectively serialized + // Otherwise it means that provided root_count is incorrect + if (root_count < 1 || root_count > decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid root count"); + } + + std::vector root_indexes(root_count); + for (int i = 0; i < root_count; ++i) { + TRY_RESULT_ASSIGN(root_indexes[i], read_uint(bit_reader, 32)); + } + + // Read number of nodes from header + TRY_RESULT(node_count, read_uint(bit_reader, 32)); + if (node_count < 1) { + return td::Status::Error("BOC decompression failed: invalid node count"); + } + + // We assume that each cell should take at least 1 byte, even effectively serialized + // Otherwise it means that provided node_count is incorrect + if (node_count > decompressed_size) { + return td::Status::Error("BOC decompression failed: incorrect node count provided"); + } + + + // Validate root indexes + for (int i = 0; i < root_count; ++i) { + if (root_indexes[i] >= node_count) { + return td::Status::Error("BOC decompression failed: invalid root index"); + } + } + + // Initialize data structures + std::vector cell_data_length(node_count), is_data_small(node_count); + std::vector is_special(node_count), cell_refs_cnt(node_count); + std::vector prunned_branch_level(node_count, 0); + + std::vector cell_builders(node_count); + std::vector> boc_graph(node_count); + + // Read cell metadata + for (int i = 0; i < node_count; ++i) { + // Check enough bits for cell type and refs count + if (bit_reader.size() < 8) { + return td::Status::Error("BOC decompression failed: not enough bits for cell metadata"); + } + + size_t cell_type = bit_reader.bits().get_uint(4); + is_special[i] = bool(cell_type); + if (is_special[i]) { + prunned_branch_level[i] = cell_type - 1; + } + bit_reader.advance(4); + + cell_refs_cnt[i] = bit_reader.bits().get_uint(4); + bit_reader.advance(4); + if (cell_refs_cnt[i] > 4) { + return td::Status::Error("BOC decompression failed: invalid cell refs count"); + } + + if (prunned_branch_level[i]) { + size_t coef = std::bitset<4>(prunned_branch_level[i]).count(); + cell_data_length[i] = (256 + 16) * coef; + } else { + // Check enough bits for data length metadata + if (bit_reader.size() < 8) { + return td::Status::Error("BOC decompression failed: not enough bits for data length"); + } + + is_data_small[i] = bit_reader.bits().get_uint(1); + bit_reader.advance(1); + cell_data_length[i] = bit_reader.bits().get_uint(7); + bit_reader.advance(7); + + if (!is_data_small[i]) { + cell_data_length[i] *= 8; + if (!cell_data_length[i]) { + cell_data_length[i] += 1024; + } + } + } + + // Validate cell data length + if (cell_data_length[i] > kMaxCellDataLengthBits) { + return td::Status::Error("BOC decompression failed: invalid cell data length"); + } + } + + // Read direct edge connections + for (int i = 0; i < node_count; ++i) { + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + TRY_RESULT(edge_connection, read_uint(bit_reader, 1)); + if (edge_connection) { + boc_graph[i][j] = i + 1; + } + } + } + + // Read initial cell data + for (int i = 0; i < node_count; ++i) { + if (prunned_branch_level[i]) { + cell_builders[i].store_long((1 << 8) + prunned_branch_level[i], 16); + } + + size_t remainder_bits = cell_data_length[i] % 8; + if (bit_reader.size() < remainder_bits) { + return td::Status::Error("BOC decompression failed: not enough bits for initial cell data"); + } + cell_builders[i].store_bits(bit_reader.subslice(0, remainder_bits)); + bit_reader.advance(remainder_bits); + cell_data_length[i] -= remainder_bits; + } + + // Decode remaining edge connections + for (size_t i = 0; i < node_count; ++i) { + if (node_count <= i + 3) { + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + if (!boc_graph[i][j]) { + boc_graph[i][j] = i + 2; + } + } + continue; + } + + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + if (!boc_graph[i][j]) { + size_t pref_size = (orig_size - bit_reader.size()); + size_t required_bits = 1 + (31 ^ td::count_leading_zeroes32(node_count - i - 3)); + + if (required_bits < 8 - (pref_size + 1) % 8 + 1) { + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, required_bits)); + boc_graph[i][j] += i + 2; + } else { + TRY_RESULT(edge_connection, read_uint(bit_reader, 1)); + if (edge_connection) { + pref_size = (orig_size - bit_reader.size()); + size_t available_bits = 8 - pref_size % 8; + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, available_bits)); + boc_graph[i][j] += i + 2; + } else { + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, required_bits)); + boc_graph[i][j] += i + 2; + } + } + } + } + } + + // Check if all graph connections are valid + for (int node = 0; node < node_count; ++node) { + for (int j = 0; j < cell_refs_cnt[node]; ++j) { + size_t child_node = boc_graph[node][j]; + if (child_node >= node_count) { + return td::Status::Error("BOC decompression failed: invalid graph connection"); + } + if (child_node <= node) { + return td::Status::Error("BOC decompression failed: circular reference in graph"); + } + } + } + + // Align to byte boundary + while ((orig_size - bit_reader.size()) % 8) { + TRY_RESULT(bit, read_uint(bit_reader, 1)); + } + + // Read remaining cell data + for (int i = 0; i < node_count; ++i) { + size_t padding_bits = 0; + if (!prunned_branch_level[i] && !is_data_small[i]) { + while (bit_reader.size() > 0 && bit_reader.bits()[0] == 0) { + ++padding_bits; + bit_reader.advance(1); + } + TRY_RESULT(bit, read_uint(bit_reader, 1)); + ++padding_bits; + } + if (cell_data_length[i] < padding_bits) { + return td::Status::Error("BOC decompression failed: invalid cell data length"); + } + size_t remaining_data_bits = cell_data_length[i] - padding_bits; + if (bit_reader.size() < remaining_data_bits) { + return td::Status::Error("BOC decompression failed: not enough bits for remaining cell data"); + } + + cell_builders[i].store_bits(bit_reader.subslice(0, remaining_data_bits)); + bit_reader.advance(remaining_data_bits); + } + + // Build cell tree + std::vector> nodes(node_count); + for (int i = node_count - 1; i >= 0; --i) { + try { + for (int child_index = 0; child_index < cell_refs_cnt[i]; ++child_index) { + size_t child = boc_graph[i][child_index]; + cell_builders[i].store_ref(nodes[child]); + } + try { + nodes[i] = cell_builders[i].finalize(is_special[i]); + } catch (vm::CellBuilder::CellWriteError& e) { + return td::Status::Error("BOC decompression failed: write error while finalizing cell."); + } + } catch (vm::VmError& e) { + return td::Status::Error("BOC decompression failed: VM error during cell construction"); + } + } + + std::vector> root_nodes; + root_nodes.reserve(root_count); + for (size_t index : root_indexes) { + root_nodes.push_back(nodes[index]); + } + + return root_nodes; +} + +td::Result boc_compress(const std::vector>& boc_roots, CompressionAlgorithm algo) { + // Check for empty input + if (boc_roots.empty()) { + return td::Status::Error("Cannot compress empty BOC roots"); + } + + td::BufferSlice compressed; + if (algo == CompressionAlgorithm::BaselineLZ4) { + TRY_RESULT_ASSIGN(compressed, boc_compress_baseline_lz4(boc_roots)); + } else if (algo == CompressionAlgorithm::ImprovedStructureLZ4) { + TRY_RESULT_ASSIGN(compressed, boc_compress_improved_structure_lz4(boc_roots)); + } else { + return td::Status::Error("Unknown compression algorithm"); + } + + td::BufferSlice compressed_with_algo(compressed.size() + 1); + compressed_with_algo.data()[0] = int(algo); + memcpy(compressed_with_algo.data() + 1, compressed.data(), compressed.size()); + return compressed_with_algo; +} + +td::Result>> boc_decompress(td::Slice compressed, int max_decompressed_size) { + if (compressed.size() == 0) { + return td::Status::Error("Can't decompress empty data"); + } + + int algo = int(compressed[0]); + compressed.remove_prefix(1); + + switch (algo) { + case int(CompressionAlgorithm::BaselineLZ4): + return boc_decompress_baseline_lz4(compressed, max_decompressed_size); + case int(CompressionAlgorithm::ImprovedStructureLZ4): + return boc_decompress_improved_structure_lz4(compressed, max_decompressed_size); + } + return td::Status::Error("Unknown compression algorithm"); +} + +} // namespace vm diff --git a/crypto/vm/boc-compression.h b/crypto/vm/boc-compression.h new file mode 100644 index 000000000..f143edd1f --- /dev/null +++ b/crypto/vm/boc-compression.h @@ -0,0 +1,42 @@ +/* +This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#pragma once + +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "vm/cells/CellSlice.h" +#include "vm/cells/CellBuilder.h" +#include "vm/excno.hpp" + +namespace vm { + +constexpr size_t kDecompressedSizeBytes = 4; + +enum class CompressionAlgorithm : int { BaselineLZ4 = 0, ImprovedStructureLZ4 = 1 }; + +td::Result boc_compress_baseline_lz4(const std::vector>& boc_roots); +td::Result>> boc_decompress_baseline_lz4(td::Slice compressed, int max_decompressed_size); + +td::Result boc_compress_improved_structure_lz4(const std::vector>& boc_roots); +td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, int max_decompressed_size); + +td::Result boc_compress(const std::vector>& boc_roots, CompressionAlgorithm algo = CompressionAlgorithm::BaselineLZ4); +td::Result>> boc_decompress(td::Slice compressed, int max_decompressed_size); + +} // namespace vm diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 32846c9ee..049ebab59 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -575,8 +575,8 @@ td::Result BagOfCells::serialize_to_impl(WriterT& writer, int mode) if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { with_hash = true; } - unsigned char buf[256]; - int s = dc->serialize(buf, 256, with_hash); + unsigned char buf[Cell::max_serialized_bytes]; + int s = dc->serialize(buf, Cell::max_serialized_bytes, with_hash); writer.store_bytes(buf, s); DCHECK(dc->size_refs() == dc_info.ref_num); // std::cerr << (dc_info.is_special() ? '*' : ' ') << i << '<' << (int)dc_info.wt << ">:"; @@ -1156,6 +1156,16 @@ td::Result CellStorageStat::add_used_storage(Ref CellStorageStat::add_used_storage(td::Span> cells, bool kill_dup, + unsigned skip_count_root) { + CellInfo result; + for (const auto& cell : cells) { + TRY_RESULT(info, add_used_storage(cell, kill_dup, skip_count_root)); + result.max_merkle_depth = std::max(result.max_merkle_depth, info.max_merkle_depth); + } + return result; +} + void NewCellStorageStat::add_cell(Ref cell) { dfs(std::move(cell), true, false); } diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 16843c1be..e4b097c36 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -143,6 +143,8 @@ struct CellStorageStat { td::Result add_used_storage(const CellSlice& cs, bool kill_dup = true, unsigned skip_count_root = 0); td::Result add_used_storage(CellSlice&& cs, bool kill_dup = true, unsigned skip_count_root = 0); td::Result add_used_storage(Ref cell, bool kill_dup = true, unsigned skip_count_root = 0); + td::Result add_used_storage(td::Span> cells, bool kill_dup = true, + unsigned skip_count_root = 0); unsigned long long limit_cells = std::numeric_limits::max(); unsigned long long limit_bits = std::numeric_limits::max(); diff --git a/crypto/vm/cellops.cpp b/crypto/vm/cellops.cpp index 61ffe5c55..b76439fac 100644 --- a/crypto/vm/cellops.cpp +++ b/crypto/vm/cellops.cpp @@ -764,6 +764,15 @@ int exec_store_same(VmState* st, const char* name, int val) { return 0; } +int exec_builder_to_slice(VmState* st) { + Stack& stack = st->get_stack(); + VM_LOG(st) << "execute BTOS"; + stack.check_underflow(1); + Ref cell = stack.pop_builder().write().finalize_novm(); + stack.push_cellslice(Ref{true, NoVm(), cell}); + return 0; +} + int exec_store_const_slice(VmState* st, CellSlice& cs, unsigned args, int pfx_bits) { unsigned refs = (args >> 3) & 3; unsigned data_bits = (args & 7) * 8 + 2; @@ -866,6 +875,7 @@ void register_cell_serialize_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xcf40, 16, "STZEROES", std::bind(exec_store_same, _1, "STZEROES", 0))) .insert(OpcodeInstr::mksimple(0xcf41, 16, "STONES", std::bind(exec_store_same, _1, "STONES", 1))) .insert(OpcodeInstr::mksimple(0xcf42, 16, "STSAME", std::bind(exec_store_same, _1, "STSAME", -1))) + .insert(OpcodeInstr::mksimple(0xcf50, 16, "BTOS", exec_builder_to_slice)->require_version(12)) .insert(OpcodeInstr::mkext(0xcf80 >> 7, 9, 5, dump_store_const_slice, exec_store_const_slice, compute_len_store_const_slice)); } diff --git a/crypto/vm/cells/CellBuilder.h b/crypto/vm/cells/CellBuilder.h index 954a1ac08..4c701c7d2 100644 --- a/crypto/vm/cells/CellBuilder.h +++ b/crypto/vm/cells/CellBuilder.h @@ -85,6 +85,9 @@ class CellBuilder : public td::CntObject { Ref get_ref(unsigned idx) const { return idx < refs_cnt ? refs[idx] : Ref{}; } + td::Span> get_refs() const { + return {refs.data(), refs_cnt}; + } void reset(); bool reset_bool() { reset(); diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index f6e35c6fe..ec7eee298 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -502,8 +502,8 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { with_hash = true; } - unsigned char buf[256]; - int s = dc->serialize(buf, 256, with_hash); + unsigned char buf[Cell::max_serialized_bytes]; + int s = dc->serialize(buf, Cell::max_serialized_bytes, with_hash); writer.store_bytes(buf, s); DCHECK(dc->size_refs() == dc_info.get_ref_num()); unsigned ref_num = dc_info.get_ref_num(); diff --git a/crypto/vm/stack.hpp b/crypto/vm/stack.hpp index 6a52e4a2b..19f19111f 100644 --- a/crypto/vm/stack.hpp +++ b/crypto/vm/stack.hpp @@ -34,6 +34,7 @@ #include "vm/excno.hpp" #include "td/utils/Span.h" +#include "td/utils/StringBuilder.h" #include @@ -570,4 +571,9 @@ class Stack : public td::CntObject { namespace td { extern template class td::Cnt>; extern template class td::Ref>>; + +inline td::StringBuilder &operator<<(td::StringBuilder &sb, const vm::StackEntry &entry) { + // Use existing textual representation of StackEntry; avoid ambiguity with std::string overloads + return sb << td::Slice(entry.to_string()); +} } // namespace td diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index e1240c6d5..ae624fe03 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -611,18 +611,20 @@ void register_prng_ops(OpcodeTable& cp0) { } int exec_compute_hash(VmState* st, int mode) { - VM_LOG(st) << "execute HASH" << (mode & 1 ? 'S' : 'C') << 'U'; + VM_LOG(st) << "execute HASH" << "CSB"[mode] << 'U'; Stack& stack = st->get_stack(); std::array hash; - if (!(mode & 1)) { + if (mode == 0) { // cell auto cell = stack.pop_cell(); hash = cell->get_hash().as_array(); - } else { + } else if (mode == 1) { // slice auto cs = stack.pop_cellslice(); - vm::CellBuilder cb; + CellBuilder cb; CHECK(cb.append_cellslice_bool(std::move(cs))); - // TODO: use cb.get_hash() instead hash = cb.finalize()->get_hash().as_array(); + } else { // builder + auto cb = stack.pop_builder(); + hash = cb.write().finalize_novm()->get_hash().as_array(); } td::RefInt256 res{true}; CHECK(res.write().import_bytes(hash.data(), hash.size(), false)); @@ -1366,6 +1368,7 @@ void register_ton_crypto_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf913, 16, "SECP256K1_XONLY_PUBKEY_TWEAK_ADD", exec_secp256k1_xonly_pubkey_tweak_add)->require_version(9)) .insert(OpcodeInstr::mksimple(0xf914, 16, "P256_CHKSIGNU", std::bind(exec_p256_chksign, _1, false))->require_version(4)) .insert(OpcodeInstr::mksimple(0xf915, 16, "P256_CHKSIGNS", std::bind(exec_p256_chksign, _1, true))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf916, 16, "HASHBU", std::bind(exec_compute_hash, _1, 2))->require_version(12)) .insert(OpcodeInstr::mksimple(0xf920, 16, "RIST255_FROMHASH", exec_ristretto255_from_hash)->require_version(4)) .insert(OpcodeInstr::mksimple(0xf921, 16, "RIST255_VALIDATE", std::bind(exec_ristretto255_validate, _1, false))->require_version(4)) @@ -1820,6 +1823,7 @@ int exec_send_message(VmState* st) { Ref dest; td::RefInt256 value; td::RefInt256 user_fwd_fee, user_ihr_fee; + unsigned extra_flags_len = 0; bool have_extra_currencies = false; bool ext_msg = msg.info->prefetch_ulong(1); if (ext_msg) { // External message @@ -1837,17 +1841,23 @@ int exec_send_message(VmState* st) { } ihr_disabled = info.ihr_disabled || st->get_global_version() >= 11; dest = std::move(info.dest); - Ref extra; + Ref extra; if (!block::tlb::t_CurrencyCollection.unpack_special(info.value.write(), value, extra)) { throw VmError{Excno::unknown, "invalid message"}; } have_extra_currencies = !extra.is_null(); user_fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); - user_ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + if (st->get_global_version() >= 12) { + user_ihr_fee = td::zero_refint(); + extra_flags_len = info.extra_flags->size(); + } else { + // Legacy: extra_flags was previously ihr_fee + user_ihr_fee = block::tlb::t_Grams.as_integer(info.extra_flags); + } } bool is_masterchain = parse_addr_workchain(*my_addr) == -1 || (!ext_msg && parse_addr_workchain(*dest) == -1); - td::Ref prices_cs; + Ref prices_cs; if (st->get_global_version() >= 6) { prices_cs = tuple_index(get_unpacked_config_tuple(st), is_masterchain ? 4 : 5).as_slice(); } else { @@ -1880,7 +1890,7 @@ int exec_send_message(VmState* st) { } else { max_cells = 1 << 13; } - vm::VmStorageStat stat(max_cells); + VmStorageStat stat(max_cells); CellSlice cs = load_cell_slice(msg_cell); cs.skip_first(cs.size()); if (st->get_global_version() >= 10 && have_extra_currencies) { @@ -1961,7 +1971,8 @@ int exec_send_message(VmState* st) { bits = 4 + my_addr->size() + dest->size() + stored_grams_len(value) + 1 + 32 + 64; td::RefInt256 fwd_fee_first = (fwd_fee * prices.first_frac) >> 16; bits += stored_grams_len(fwd_fee - fwd_fee_first); - bits += stored_grams_len(ihr_fee); + // Legacy: extra_flags was previously ihr_fee + bits += st->get_global_version() >= 12 ? extra_flags_len : stored_grams_len(ihr_fee); } // init bits++; diff --git a/dht/dht-in.hpp b/dht/dht-in.hpp index 0d668d438..2f4e70416 100644 --- a/dht/dht-in.hpp +++ b/dht/dht-in.hpp @@ -47,8 +47,6 @@ class DhtMemberImpl : public DhtMember { td::uint32 k_; td::uint32 a_; td::int32 network_id_{-1}; - td::uint32 max_cache_time_ = 60; - td::uint32 max_cache_size_ = 100; std::vector buckets_; @@ -57,17 +55,14 @@ class DhtMemberImpl : public DhtMember { // to be republished once in a while std::map our_values_; - std::map cached_values_; - td::ListNode cached_values_lru_; - std::map values_; + std::set> values_ttl_order_; td::Timestamp fill_att_ = td::Timestamp::in(0); td::Timestamp republish_att_ = td::Timestamp::in(0); DhtKeyId last_republish_key_ = DhtKeyId::zero(); DhtKeyId last_check_key_ = DhtKeyId::zero(); - adnl::AdnlNodeIdShort last_check_reverse_conn_ = adnl::AdnlNodeIdShort::zero(); struct ReverseConnection { adnl::AdnlNodeIdShort dht_node_; @@ -76,6 +71,7 @@ class DhtMemberImpl : public DhtMember { }; std::map reverse_connections_; std::set our_reverse_connections_; + std::set> reverse_connections_ttl_order_; class Callback : public adnl::Adnl::Callback { public: @@ -193,6 +189,9 @@ class DhtMemberImpl : public DhtMember { } void get_self_node(td::Promise promise) override; + + static constexpr size_t MAX_VALUES = 100000; + static constexpr size_t MAX_REVERSE_CONNECTIONS = 100000; }; } // namespace dht diff --git a/dht/dht.cpp b/dht/dht.cpp index b774b5f91..da3b33601 100644 --- a/dht/dht.cpp +++ b/dht/dht.cpp @@ -228,25 +228,23 @@ void DhtMemberImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::dht_findVa td::Promise promise) { find_value_queries_++; auto it = values_.find(DhtKeyId{query.key_}); - if (it != values_.end() && it->second.expired()) { - values_.erase(it); - it = values_.end(); - } - if (it != values_.end()) { + if (it != values_.end() && !it->second.expired()) { promise.set_value(create_serialize_tl_object(it->second.tl())); return; } auto k = static_cast(query.k_); - if (k > max_k()) { - k = max_k(); - } + k = std::min(k, max_k()); auto R = get_nearest_nodes(DhtKeyId{query.key_}, k); promise.set_value(create_serialize_tl_object(R.tl())); } td::Status DhtMemberImpl::store_in(DhtValue value) { + if (value.ttl() > (td::uint32)td::Clocks::system() + 3600 + 60) { + // clients typically set ttl = 1 hour + return td::Status::Error("ttl is too big"); + } if (value.expired()) { VLOG(DHT_INFO) << this << ": dropping expired value: " << value.key_id() << " expire_at = " << value.ttl(); return td::Status::OK(); @@ -259,10 +257,12 @@ td::Status DhtMemberImpl::store_in(DhtValue value) { if (dist < k_ + 10) { auto it = values_.find(key_id); if (it != values_.end()) { + CHECK(values_ttl_order_.erase({it->second.ttl(), it->first})); it->second.update(std::move(value)); } else { - values_.emplace(key_id, std::move(value)); + it = values_.emplace(key_id, std::move(value)).first; } + CHECK(values_ttl_order_.insert({it->second.ttl(), it->first}).second); } else { VLOG(DHT_INFO) << this << ": dropping too remote value: " << value.key_id() << " distance = " << dist; } @@ -324,7 +324,13 @@ void DhtMemberImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::dht_regist TRY_RESULT_PROMISE(promise, encryptor, pub.create_encryptor()); TRY_STATUS_PROMISE(promise, encryptor->check_signature(to_sign, query.signature_)); DhtKeyId key_id = get_reverse_connection_key(client_id).compute_key_id(); - reverse_connections_[client_id] = ReverseConnection{src, key_id, td::Timestamp::at_unix(std::min(ttl, now + 300))}; + auto it = reverse_connections_.find(client_id); + if (it != reverse_connections_.end()) { + CHECK(reverse_connections_ttl_order_.erase({it->second.ttl_, client_id})); + } + auto &entry = reverse_connections_[client_id] = + ReverseConnection{src, key_id, td::Timestamp::at_unix(std::min(ttl, now + 300))}; + CHECK(reverse_connections_ttl_order_.insert({entry.ttl_, client_id}).second); promise.set_value(create_serialize_tl_object()); } @@ -332,25 +338,18 @@ void DhtMemberImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::dht_reques td::Promise promise) { adnl::AdnlNodeIdShort client{query.client_}; auto it = reverse_connections_.find(client); - if (it != reverse_connections_.end()) { - if (it->second.ttl_.is_in_past()) { - reverse_connections_.erase(it); - } else { - PublicKey pub{query.target_->id_}; - TRY_RESULT_PROMISE(promise, encryptor, pub.create_encryptor()); - TRY_STATUS_PROMISE(promise, - encryptor->check_signature(serialize_tl_object(query.target_, true), query.signature_)); - td::actor::send_closure(adnl_, &adnl::Adnl::send_message, id_, it->second.dht_node_, - create_serialize_tl_object( - std::move(query.target_), std::move(query.signature_), query.client_)); - promise.set_result(create_serialize_tl_object()); - return; - } + if (it != reverse_connections_.end() && !it->second.ttl_.is_in_past()) { + PublicKey pub{query.target_->id_}; + TRY_RESULT_PROMISE(promise, encryptor, pub.create_encryptor()); + TRY_STATUS_PROMISE(promise, encryptor->check_signature(serialize_tl_object(query.target_, true), query.signature_)); + td::actor::send_closure(adnl_, &adnl::Adnl::send_message, id_, it->second.dht_node_, + create_serialize_tl_object( + std::move(query.target_), std::move(query.signature_), query.client_)); + promise.set_result(create_serialize_tl_object()); + return; } auto k = static_cast(query.k_); - if (k > max_k()) { - k = max_k(); - } + k = std::min(k, max_k()); auto R = get_nearest_nodes(get_reverse_connection_key(client).compute_key_id(), k); promise.set_value(create_serialize_tl_object(R.tl())); } @@ -434,7 +433,7 @@ void DhtMemberImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice d auto S = [&]() -> td::Status { auto f = F.move_as_ok(); adnl::AdnlNodeIdShort client{f->client_}; - if (!our_reverse_connections_.count(client)) { + if (!our_reverse_connections_.contains(client)) { return td::Status::Error(PSTRING() << ": unknown id for reverse ping: " << client); } TRY_RESULT_PREFIX(node, adnl::AdnlNode::create(f->target_), "failed to parse node: "); @@ -501,7 +500,8 @@ void DhtMemberImpl::register_reverse_connection(adnl::AdnlNodeIdFull client, td: td::actor::send_closure(keyring_, &keyring::Keyring::sign_message, client_short.pubkey_hash(), register_reverse_connection_to_sign(client_short, id_, ttl), [=, print_id = print_id(), list = get_nearest_nodes(key_id, k_ * 2), SelfId = actor_id(this), - promise = std::move(promise)](td::Result R) mutable { + promise = std::move(promise), id = id_, k = k_, a = a_, network_id = network_id_, + client_only = client_only_, adnl = adnl_](td::Result R) mutable { TRY_RESULT_PROMISE_PREFIX(promise, signature, std::move(R), "Failed to sign: "); td::actor::send_closure(SelfId, &DhtMemberImpl::get_self_node, [=, list = std::move(list), signature = std::move(signature), @@ -509,8 +509,8 @@ void DhtMemberImpl::register_reverse_connection(adnl::AdnlNodeIdFull client, td: R.ensure(); td::actor::create_actor( "RegisterReverseQuery", key_id, std::move(client), ttl, - std::move(signature), print_id, id_, std::move(list), k_, a_, - network_id_, R.move_as_ok(), client_only_, SelfId, adnl_, + std::move(signature), print_id, id, std::move(list), k, a, + network_id, R.move_as_ok(), client_only, SelfId, adnl, std::move(promise)) .release(); }); @@ -534,25 +534,22 @@ void DhtMemberImpl::request_reverse_ping(adnl::AdnlNode target, adnl::AdnlNodeId void DhtMemberImpl::request_reverse_ping_cont(adnl::AdnlNode target, td::BufferSlice signature, adnl::AdnlNodeIdShort client, td::Promise promise) { auto it = reverse_connections_.find(client); - if (it != reverse_connections_.end()) { - if (it->second.ttl_.is_in_past()) { - reverse_connections_.erase(it); - } else { - td::actor::send_closure(adnl_, &adnl::Adnl::send_message, id_, it->second.dht_node_, - create_serialize_tl_object( - target.tl(), std::move(signature), client.bits256_value())); - promise.set_result(td::Unit()); - return; - } + if (it != reverse_connections_.end() && !it->second.ttl_.is_in_past()) { + td::actor::send_closure(adnl_, &adnl::Adnl::send_message, id_, it->second.dht_node_, + create_serialize_tl_object( + target.tl(), std::move(signature), client.bits256_value())); + promise.set_result(td::Unit()); + return; } auto key_id = get_reverse_connection_key(client).compute_key_id(); get_self_node([=, target = std::move(target), signature = std::move(signature), promise = std::move(promise), SelfId = actor_id(this), print_id = print_id(), list = get_nearest_nodes(key_id, k_ * 2), - client_only = client_only_](td::Result R) mutable { + client_only = client_only_, id = id_, k = k_, a = a_, adnl = adnl_, + network_id = network_id_](td::Result R) mutable { R.ensure(); td::actor::create_actor( - "RequestReversePing", client, std::move(target), std::move(signature), print_id, id_, std::move(list), k_, a_, - network_id_, R.move_as_ok(), client_only, SelfId, adnl_, std::move(promise)) + "RequestReversePing", client, std::move(target), std::move(signature), print_id, id, std::move(list), k, a, + network_id, R.move_as_ok(), client_only, SelfId, adnl, std::move(promise)) .release(); }); } @@ -561,6 +558,9 @@ void DhtMemberImpl::check() { VLOG(DHT_INFO) << this << ": ping=" << ping_queries_ << " fnode=" << find_node_queries_ << " fvalue=" << find_value_queries_ << " store=" << store_queries_ << " addrlist=" << get_addr_list_queries_; + VLOG(DHT_INFO) << this << ": values=" << values_.size() << " our_values=" << our_values_.size(); + VLOG(DHT_INFO) << this << ": reverse_conns=" << reverse_connections_.size() + << " our_reverse_conns=" << our_reverse_connections_.size(); for (auto &bucket : buckets_) { bucket.check(client_only_, adnl_, actor_id(this), id_); } @@ -568,10 +568,15 @@ void DhtMemberImpl::check() { save_to_db(); } - if (values_.size() > 0) { + for (auto it = values_ttl_order_.begin(); + it != values_ttl_order_.end() && (it->first < td::Clocks::system() || values_.size() > MAX_VALUES);) { + CHECK(values_.erase(it->second)); + it = values_ttl_order_.erase(it); + } + if (!values_.empty()) { auto it = values_.lower_bound(last_check_key_); if (it != values_.end() && it->first == last_check_key_) { - it++; + ++it; } if (it == values_.end()) { it = values_.begin(); @@ -579,14 +584,10 @@ void DhtMemberImpl::check() { td::uint32 cnt = 0; auto s = last_check_key_; - while (values_.size() > 0 && cnt < 1 && it->first != s) { + while (!values_.empty() && cnt < 1 && it->first != s) { last_check_key_ = it->first; cnt++; - if (it->second.expired()) { - it = values_.erase(it); - - // do not republish soon-to-be-expired values - } else if (it->second.ttl() > td::Clocks::system() + 60) { + if (it->second.ttl() > td::Clocks::system() + 60) { auto dist = distance(it->first, k_ + 10); if (dist == 0) { @@ -598,16 +599,17 @@ void DhtMemberImpl::check() { }); send_store(it->second.clone(), std::move(P)); } - it++; + ++it; } else if (dist >= k_ + 10) { + CHECK(values_ttl_order_.erase({it->second.ttl(), it->first})); it = values_.erase(it); } else { - it++; + ++it; } } else { - it++; + ++it; } - if (values_.size() == 0) { + if (values_.empty()) { break; } if (it == values_.end()) { @@ -615,21 +617,17 @@ void DhtMemberImpl::check() { } } } - if (reverse_connections_.size() > 0) { - auto it = reverse_connections_.upper_bound(last_check_reverse_conn_); - if (it == reverse_connections_.end()) { - it = reverse_connections_.begin(); - } - last_check_reverse_conn_ = it->first; - if (it->second.ttl_.is_in_past()) { - reverse_connections_.erase(it); - } + for (auto it = reverse_connections_ttl_order_.begin(); + it != reverse_connections_ttl_order_.end() && + (it->first.is_in_past() || reverse_connections_.size() > MAX_REVERSE_CONNECTIONS);) { + CHECK(reverse_connections_.erase(it->second)); + it = reverse_connections_ttl_order_.erase(it); } if (republish_att_.is_in_past()) { auto it = our_values_.lower_bound(last_republish_key_); if (it != our_values_.end() && it->first == last_republish_key_) { - it++; + ++it; } if (it == our_values_.end()) { it = our_values_.begin(); @@ -705,8 +703,8 @@ void DhtMemberImpl::send_store(DhtValue value, td::Promise promise) { } void DhtMemberImpl::get_self_node(td::Promise promise) { - auto P = td::PromiseCreator::lambda([promise = std::move(promise), print_id = print_id(), id = id_, - keyring = keyring_, client_only = client_only_, + auto P = td::PromiseCreator::lambda([promise = std::move(promise), id = id_, keyring = keyring_, + client_only = client_only_, network_id = network_id_](td::Result R) mutable { R.ensure(); auto node = R.move_as_ok(); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index ad5eba948..79dbc6bcc 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -144,6 +144,7 @@ Example: if the last masterchain block seqno is `19071` then the list contains b - Fix recursive jump to continuations with non-null control data. ## Version 10 +__Enabled in mainnet on 2025-05-07__ ### Extra currencies - Internal messages cannot carry more than 2 different extra currencies. The limit can be changed in size limits config (`ConfigParam 43`). @@ -194,6 +195,7 @@ Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" - Exceeding state limits in transaction now reverts `end_lt` back to `start_lt + 1` and collects action fines. ## Version 11 +__Enabled in mainnet on 2025-07-05__ ### c7 tuple **c7** tuple extended from 17 to 18 elements: @@ -225,4 +227,49 @@ This is required to help computing storage stats in the future, after collator-v ### Other changes - Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. -- In new internal messages `ihr_disabled` is automatically set to `1`, `ihr_fee` is always zero. \ No newline at end of file +- In new internal messages `ihr_disabled` is automatically set to `1`, `ihr_fee` is always zero. + +## Version 12 + +### Extra message flags and new bounce format +Field `ihr_fee:Grams` in internal message is now called `extra_flags:(VarUInteger 16)` (it's the same format). +This field does not represent fees. `ihr_fee` is always zero since version 11, so this field was essentially unused. + +`(extra_flags & 1) = 1` enables the new bounce format for the message. The bounced message contains information about the transaction. +If `(extra_flags & 3) = 3`, the bounced message contains the whole body of the original message. Otherwise, only the bits from the root of the original body are returned. +When the message with new bounce flag is bounced, the bounced message body has the following format (`new_bounce_body`): +``` +_ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; +_ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; + +new_bounce_body#fffffffe + original_body:^Cell + original_info:^NewBounceOriginalInfo + bounced_by_phase:uint8 exit_code:int32 + compute_phase:(Maybe NewBounceComputePhaseInfo) + = NewBounceBody; +``` +- `original_body` - cell that contains the body of the original message. If `extra_flags & 2` then the whole body is returned, otherwise it is only the root without refs. +- `original_info` - value, lt and unixtime of the original message. +- `bounced_by_phase`: + - `0` - compute phase was skipped. `exit_code` denotes the skip reason: + - `exit_code = -1` - no state (account is uninit or frozen, and no state init is present in the message). + - `exit_code = -2` - bad state (account is uninit or frozen, and state init in the message has the wrong hash). + - `exit_code = -3` - no gas. + - `exit_code = -4` - account is suspended. + - `1` - compute phase failed. `exit_code` is the value from the compute phase. + - `2` - action phase failed. `exit_code` is the value from the action phase. +- `exit_code` - 32-bit exit code, see above. +- `compute_phase` - exists if it was not skipped (`bounced_by_phase > 0`): + - `gas_used`, `vm_steps` - same as in `TrComputePhase` of the transaction. + +### New TVM instructions +- `BTOS` (`b - s`) - same as `ENDC CTOS`, but without gas cost for cell creation and loading. Gas cost: `26`. +- `HASHBU` (`b - hash`) - same as `ENDC HASHCU`, but without gas cost for cell creation. Gas cost: `26`. + +### Other TVM changes +- `SENDMSG` instruction treats `extra_flags` field accordingly (see above). + +### Other changes +- Account size in masterchain is now limited to `2048` cells. This can be configured in size limits config (`ConfigParam 43`). + - The previous limit was the same as in basechain (`65536`). \ No newline at end of file diff --git a/emulator/test/emulator-tests.cpp b/emulator/test/emulator-tests.cpp index ae273ddfd..4057f5816 100644 --- a/emulator/test/emulator-tests.cpp +++ b/emulator/test/emulator-tests.cpp @@ -221,7 +221,7 @@ TEST(Emulator, wallet_int_and_ext_msg) { { vm::CellBuilder cb; block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(0)); - msg_info.ihr_fee = cb.as_cellslice_ref(); + msg_info.extra_flags = cb.as_cellslice_ref(); } msg_info.created_lt = 0; msg_info.created_at = static_cast(utime); diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp index fd624ff8c..d6b798222 100644 --- a/lite-client/ext-client.cpp +++ b/lite-client/ext-client.cpp @@ -104,7 +104,7 @@ class ExtClientImpl : public ExtClient { promise.set_result(std::move(R)); }; LOG(DEBUG) << "Sending query " << query_info.to_str() << " to server #" << server.idx << " (" - << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + << server.config.hostname << ")"; send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, std::move(P)); } @@ -173,9 +173,9 @@ class ExtClientImpl : public ExtClient { td::actor::ActorId parent_; size_t idx_; }; - LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" - << server.config.addr.get_port() << ") for query " << (query_info ? query_info->to_str() : "[none]"); - server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.hostname << ") for query " + << (query_info ? query_info->to_str() : "[none]"); + server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.hostname, std::make_unique(actor_id(this), server_idx)); } @@ -204,8 +204,7 @@ class ExtClientImpl : public ExtClient { continue; } if (server.timeout.is_in_past()) { - LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() - << ":" << server.config.addr.get_port() << ")"; + LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.hostname << ")"; server.client.reset(); server.alive = false; server.ignore_until = {}; diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index b4cc2709d..9e260dea0 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -108,7 +108,7 @@ void TestNode::run() { if (single_liteserver_idx_ != -1) { // Use single liteserver from config CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < servers.size()); td::TerminalIO::out() << "using liteserver #" << single_liteserver_idx_ << " with addr " - << servers[single_liteserver_idx_].addr << "\n"; + << servers[single_liteserver_idx_].hostname << "\n"; servers = {servers[single_liteserver_idx_]}; } } diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index b46d46a5c..7c9e7cfc7 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -318,16 +318,25 @@ bool LiteServerConfig::Slice::accepts_query(const QueryInfo& query_info) const { td::Result> LiteServerConfig::parse_global_config( const ton_api::liteclient_config_global& config) { std::vector servers; + auto get_hostname = [](const auto& f) -> std::string { + if (f->hostname_.empty()) { + return PSTRING() << td::IPAddress::ipv4_to_str(f->ip_) << ":" << f->port_; + } + if (f->port_ == 0) { + return f->hostname_; + } + return PSTRING() << f->hostname_ << ":" << f->port_; + }; for (const auto& f : config.liteservers_) { LiteServerConfig server; - TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.hostname = get_hostname(f); server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; server.is_full = true; servers.push_back(std::move(server)); } for (const auto& f : config.liteservers_v2_) { LiteServerConfig server; - TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.hostname = get_hostname(f); server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; server.is_full = false; for (const auto& slice_obj : f->slices_) { diff --git a/lite-client/query-utils.hpp b/lite-client/query-utils.hpp index 28500e266..d21424695 100644 --- a/lite-client/query-utils.hpp +++ b/lite-client/query-utils.hpp @@ -73,11 +73,14 @@ struct LiteServerConfig { public: ton::adnl::AdnlNodeIdFull adnl_id; - td::IPAddress addr; + std::string hostname; LiteServerConfig() = default; - LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress addr) - : is_full(true), adnl_id(adnl_id), addr(addr) { + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, std::string hostname) + : is_full(true), adnl_id(adnl_id), hostname(std::move(hostname)) { + } + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress ip) + : is_full(true), adnl_id(adnl_id), hostname(PSTRING() << ip.get_ip_str() << ":" << ip.get_port()) { } bool accepts_query(const QueryInfo& query_info) const; diff --git a/lite-server-daemon/lite-server-config.hpp b/lite-server-daemon/lite-server-config.hpp index 627eb7156..cf7ea4512 100644 --- a/lite-server-daemon/lite-server-config.hpp +++ b/lite-server-daemon/lite-server-config.hpp @@ -192,7 +192,7 @@ class Config { std::vector> lite_slaves_vec; for (auto &x : liteslaves) { lite_slaves_vec.push_back( - ton::create_tl_object(x.key.tl(), x.addr.get_ipv4(), x.addr.get_port())); + ton::create_tl_object(x.key.tl(), x.addr.get_ipv4(), x.addr.get_port(), "")); } return ton::create_tl_object( diff --git a/lite-server-daemon/lite-server-daemon.cpp b/lite-server-daemon/lite-server-daemon.cpp index f2eac8bd4..84b044d50 100644 --- a/lite-server-daemon/lite-server-daemon.cpp +++ b/lite-server-daemon/lite-server-daemon.cpp @@ -168,10 +168,6 @@ class LiteServerDaemon : public td::actor::Actor { void new_key_block(ton::validator::BlockHandle handle) override { } - void send_validator_telemetry(ton::PublicKeyHash key, - ton::tl_object_ptr telemetry) override { - } - Callback(td::actor::ActorId id) : id_(id) { } diff --git a/recent_changelog.md b/recent_changelog.md index 6b65d2183..fc4ae86c4 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,3 +1,9 @@ -## 2025.07 Accelerator Update +## 2025.10 Update -Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators) +1. [TVM version v12](./doc/GlobalVersions.md): full bounces, new `BTOS` and `HASHBU` instuctions, limit on contract size in masterchain. +2. Optimistic collation/validation: allow nodes to generate and check block candidates before previous block is fully signed (not fully activated yet). +3. Introduced custom block compression algorithm. +4. Overlay improvements: improved overlay discovery on shard configuration update, private externals in custom overlays. +5. Various improvements: session stats, telemetry in fast-sync overlay, earlier block broadcasts, limiting ttl for values in DHT, fixing search by utime in native blockexplorer, faster downloading candidates in validator session, parallelization of storing to cell_db, avoiding touching packfiles on startup. + +Besides the work of the core team, this update is based on the efforts of the Tonstudio team: @hacker-volodya @Shvandre; and @mkiesel (avoiding touching packfiles on startup). diff --git a/storage/NodeActor.cpp b/storage/NodeActor.cpp index e24fffff5..92a83968f 100644 --- a/storage/NodeActor.cpp +++ b/storage/NodeActor.cpp @@ -27,6 +27,7 @@ #include "td/utils/overloaded.h" #include "tl-utils/tl-utils.hpp" #include "auto/tl/ton_api.hpp" +#include "common/delay.h" #include "td/actor/MultiPromise.h" namespace ton { @@ -545,8 +546,9 @@ void NodeActor::loop_queries() { auto it = peers_.find(part.peer_id); CHECK(it != peers_.end()); auto &state = it->second.state; - CHECK(state->peer_state_ready_); - CHECK(state->peer_state_.load().will_upload); + if (!state->peer_state_ready_ || !state->peer_state_.load().will_upload) { + continue; + } CHECK(state->node_queries_active_.size() < MAX_PEER_TOTAL_QUERIES); auto part_id = part.part_id; if (state->node_queries_active_.insert(static_cast(part_id)).second) { @@ -787,15 +789,21 @@ void NodeActor::db_store_torrent_meta() { return; } next_db_store_meta_at_ = td::Timestamp::never(); - auto meta = torrent_.get_meta_str(); - db_->set(create_hash_tl_object(torrent_.get_hash()), td::BufferSlice(meta), - [new_count = (td::int64)torrent_.get_ready_parts_count(), SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, new_count); - } - }); + auto meta = torrent_.get_meta(); + delay_action( + [SelfId = actor_id(this), meta = std::move(meta), db = db_, hash = torrent_.get_hash(), + new_count = (td::int64)torrent_.get_ready_parts_count()]() { + auto meta_str = meta.serialize(); + db->set(create_hash_tl_object(hash), td::BufferSlice(meta_str), + [=](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, new_count); + } + }); + }, + td::Timestamp::now()); } void NodeActor::after_db_store_torrent_meta(td::Result R) { diff --git a/storage/storage-daemon/storage-daemon.cpp b/storage/storage-daemon/storage-daemon.cpp index 98388428d..d1cbf37b5 100644 --- a/storage/storage-daemon/storage-daemon.cpp +++ b/storage/storage-daemon/storage-daemon.cpp @@ -42,6 +42,8 @@ #if TD_DARWIN || TD_LINUX #include #endif +#include "td/utils/port/rlimit.h" + #include namespace ton { @@ -950,6 +952,7 @@ int main(int argc, char *argv[]) { SCOPE_EXIT { td::log_interface = td::default_log_interface; }; + LOG_STATUS(td::change_maximize_rlimit(td::RlimitType::nofile, 786432)); td::IPAddress ip_addr; bool client_mode = false; diff --git a/tdutils/td/utils/StringBuilder.h b/tdutils/td/utils/StringBuilder.h index 685416fe3..0edb4289d 100644 --- a/tdutils/td/utils/StringBuilder.h +++ b/tdutils/td/utils/StringBuilder.h @@ -164,4 +164,9 @@ inline LambdaPrintHelper operator<<(td::StringBuilder& sb, co return LambdaPrintHelper{sb}; } +// Added to avoid ambiguity when streaming std::string into StringBuilder +inline StringBuilder &operator<<(StringBuilder &sb, const std::string &str) { + return sb << Slice(str); +} + } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index d787f1ca2..de793aec0 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -81,6 +81,52 @@ class ThreadCpuTimer { bool is_paused_{false}; }; +class RealCpuTimer { + public: + RealCpuTimer() : RealCpuTimer(false) { + } + explicit RealCpuTimer(bool is_paused) : real_(is_paused), cpu_(is_paused) { + } + RealCpuTimer(const RealCpuTimer &other) = default; + RealCpuTimer &operator=(const RealCpuTimer &other) = default; + + double elapsed_real() const { + return real_.elapsed(); + } + double elapsed_cpu() const { + return cpu_.elapsed(); + } + struct Time { + double real = 0.0, cpu = 0.0; + double get(bool is_cpu) const { + return is_cpu ? cpu : real; + } + Time &operator+=(const Time &other) { + real += other.real; + cpu += other.cpu; + return *this; + } + Time operator-(const Time &other) const { + return {.real = real - other.real, .cpu = cpu - other.cpu}; + } + }; + Time elapsed_both() const { + return {.real = real_.elapsed(), .cpu = cpu_.elapsed()}; + } + void pause() { + real_.pause(); + cpu_.pause(); + } + void resume() { + real_.resume(); + cpu_.resume(); + } + + private: + Timer real_; + ThreadCpuTimer cpu_; +}; + class PerfLog; struct EmptyDeleter { template @@ -114,6 +160,9 @@ class PerfLog { }; template double PerfLogAction::finish(const T &result) { + if (!perf_log_) { + return 0.0; + } if (result.is_ok()) { return perf_log_->finish_action(i_, td::Status::OK()); } else { diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index dc424c7f0..e8ec3ab5b 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -381,9 +381,6 @@ class TestNode : public td::actor::Actor { void new_key_block(ton::validator::BlockHandle handle) override { } - void send_validator_telemetry(ton::PublicKeyHash key, - ton::tl_object_ptr telemetry) override { - } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, diff --git a/third-party/pybind11 b/third-party/pybind11 index be97c5a98..326200019 160000 --- a/third-party/pybind11 +++ b/third-party/pybind11 @@ -1 +1 @@ -Subproject commit be97c5a98b4b252c524566f508b5c79410d118c6 +Subproject commit 3262000195616afedaf95185f1fe965c7eb82be6 diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 0d0d0458a..4ec393f58 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -344,6 +344,8 @@ validatorSession.candidateId src:int256 root_hash:int256 file_hash:int256 collat validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Message) state:int = validatorSession.BlockUpdate; validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate; validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate; +validatorSession.compressedCandidateV2 flags:# src:int256 round:int root_hash:int256 data:bytes = validatorSession.Candidate; +validatorSession.optimisticCandidateBroadcast flags:# prev_candidate_id:int256 data:bytes = validatorSession.OptimisticCandidateBroadcast; validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config; @@ -431,6 +433,8 @@ tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_ha proof:bytes data:bytes = tonNode.Broadcast; tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int flags:# compressed:bytes = tonNode.Broadcast; +tonNode.blockBroadcastCompressedV2 id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + signatures:(vector tonNode.blockSignature) flags:# compressed:bytes = tonNode.Broadcast; tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast; tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast; tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; @@ -439,6 +443,8 @@ tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int vali collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast; tonNode.newBlockCandidateBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; +tonNode.newBlockCandidateBroadcastCompressedV2 id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; // optimistic broadcast of response to tonNode.getOutMsgQueueProof with dst_shard, block and limits arguments tonNode.outMsgQueueProofBroadcast dst_shard:tonNode.shardId block:tonNode.blockIdExt @@ -457,6 +463,7 @@ ton.blockIdApprove root_cell_hash:int256 file_hash:int256 = ton.BlockId; tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullCompressed id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull; +tonNode.dataFullCompressedV2 id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullEmpty = tonNode.DataFull; tonNode.capabilities#f5bf60c0 version_major:int version_minor:int flags:# = tonNode.Capabilities; @@ -651,8 +658,8 @@ liteserver.descV2.sliceSimple shards:(vector tonNode.shardId) = liteserver.descV liteserver.descV2.shardInfo shard_id:tonNode.shardId seqno:int utime:int lt:long = liteserver.descV2.ShardInfo; liteserver.descV2.sliceTimed shards_from:(vector liteserver.descV2.shardInfo) shards_to:(vector liteserver.descV2.shardInfo) = liteserver.descV2.Slice; -liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteserver.descV2 id:PublicKey ip:int port:int slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; +liteserver.desc id:PublicKey ip:int port:int hostname:string = liteserver.Desc; +liteserver.descV2 id:PublicKey ip:int port:int hostname:string slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; liteclient.config.global liteservers:(vector liteserver.desc) liteservers_v2:(vector liteserver.descV2) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; @@ -692,7 +699,7 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int block_sender:Bool = engine.validator.CustomOverlayNode; engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) sender_shards:(vector tonNode.shardId) - = engine.validator.CustomOverlay; + skip_public_msg_send:Bool = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; engine.liteserver.config ip:int out_port:int adnl:(vector engine.adnl) @@ -785,7 +792,7 @@ engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSiz engine.validator.exportedPrivateKeys encrypted_data:bytes = engine.validator.ExportedPrivateKeys; engine.validator.collationManagerStats.shard shard_id:tonNode.shardId self_collate:Bool select_mode:string active:Bool collators:(vector int256) = engine.validator.collationManagerStats.Shard; -engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string = engine.validator.collationManagerStats.Collator; +engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string banned_for:double = engine.validator.collationManagerStats.Collator; engine.validator.collationManagerStats.localId adnl_id:int256 shards:(vector engine.validator.collationManagerStats.shard) collators:(vector engine.validator.collationManagerStats.collator) = engine.validator.collationManagerStats.LocalId; engine.validator.collationManagerStats local_ids:(vector engine.validator.collationManagerStats.localId) = engine.validator.CollationManagerStats; @@ -961,21 +968,29 @@ validatorStats.blockStats ext_msgs:validatorStats.blockStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) old_out_msg_queue_size:long new_out_msg_queue_size:long msg_queue_cleaned:int neighbors:(vector validatorStats.blockStats.neighborStats) = validatorStats.BlockStats; +validatorStats.storageStatCacheStats + small_cnt:long small_cells:long hit_cnt:long hit_cells:long miss_cnt:long miss_cells:long = validatorStats.StorageStatCacheStats; validatorStats.collatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double bytes:int collated_data_bytes:int attempt:int self:int256 is_validator:Bool total_time:double work_time:double cpu_work_time:double time_stats:string + work_time_real_stats:string + work_time_cpu_stats:string block_limits:validatorStats.blockLimitsStatus - block_stats:validatorStats.blockStats = validatorSession.stats.CollatedBlock; + block_stats:validatorStats.blockStats + storage_stat_cache:validatorStats.storageStatCacheStats = validatorSession.stats.CollatedBlock; validatorStats.validatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double self:int256 valid:Bool comment:string bytes:int collated_data_bytes:int - total_time:double work_time:double cpu_work_time:double = validatorStats.ValidatedBlock; + total_time:double work_time:double cpu_work_time:double time_stats:string + work_time_real_stats:string + work_time_cpu_stats:string + storage_stat_cache:validatorStats.storageStatCacheStats = validatorStats.ValidatedBlock; validatorStats.newValidatorGroup.node id:int256 pubkey:PublicKey adnl_id:int256 weight:long = validatorStats.newValidatorGroup.Node; validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int @@ -992,7 +1007,8 @@ validatorStats.collatorNodeResponse self:int256 validator_id:int256 timestamp:do collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; -collatorNode.pong flags:# = collatorNode.Pong; +collatorNode.compressedCandidateV2 flags:# source:PublicKey id:tonNode.blockIdExt data:bytes = collatorNode.Candidate; +collatorNode.pong#5bbf0521 flags:# version:flags.0?int = collatorNode.Pong; collatorNode.error code:int message:string = collatorNode.Error; shardBlockVerifier.subscribed flags:# = shardBlockVerifier.Subscribed; @@ -1001,6 +1017,9 @@ shardBlockVerifier.confirmBlocks blocks:(vector tonNode.blockIdExt) = shardBlock ---functions--- collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; +collatorNode.generateBlockOptimistic shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) + creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; +collatorNode.requestBlockCallback flags:# block_id:tonNode.BlockIdExt = collatorNode.Candidate; collatorNode.ping flags:# = collatorNode.Pong; shardBlockVerifier.subscribe shard:tonNode.shardId flags:# = shardBlockVerifier.Subscribed; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 7984dc638..2f383fd3a 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index 0343bdf50..377f03a92 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -257,6 +257,32 @@ fun test119(a: int, b: int) { return (l, c, d); } +struct Point { + x: int + y: int +} + +struct ROLine { + readonly from: Point + readonly to: Point +} + +fun eq(v: T) { return v } + +@method_id(120) +fun test120() { + val p: Point = { x: 10, y: 20 }; + eq(p).x = 30; + return p; +} + +@method_id(121) +fun test121(l: ROLine) { + eq(l.from).x = eq(l.to).y; + eq(l.from).y += 10; + return l +} + fun main(value: int, ) { @@ -288,6 +314,8 @@ fun main(value: int, ) { @testcase | 117 | | [ 20 ] @testcase | 118 | 3 | 10 3 0 1 @testcase | 119 | 1 2 | 2 2 4 +@testcase | 120 | | 10 20 +@testcase | 121 | null | (null) @fif_codegen diff --git a/tolk-tester/tests/calls-tests.tolk b/tolk-tester/tests/calls-tests.tolk index 9405c9ead..2177136f5 100644 --- a/tolk-tester/tests/calls-tests.tolk +++ b/tolk-tester/tests/calls-tests.tolk @@ -179,7 +179,11 @@ fun test11(a: int, b: int) { return ((b < a), (b <= a), (b > a), (b >= a)); } -fun main() {} +fun main() { + // mark used to codegen them + cmp1; cmp2; cmp3; cmp4; + leq1; leq2; leq3; leq4; +} /** @testcase | 101 | 10 | 10 15 100 12 3 110 300 diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 57a92f03b..e1c2ac9c3 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -16,6 +16,16 @@ fun endCell(b: builder): cell fun beginParse(c: cell): slice asm "CTOS"; +@noinline +fun triggerOverflowIntConst() { + return beginCell().storeInt(10, 4) +} + +@noinline +fun triggerOverflowUintConst() { + return beginCell().storeUint(123, 6) +} + @method_id(101) fun test1(): [int,int,int,int,int] { var b: builder = beginCell().storeUint(1, 32); @@ -249,6 +259,7 @@ fun test18() { return (s.loadUint(14), s.loadInt(8), s.loadUint(4)); } +@method_id(119) fun test19() { // numbers with potential overflow for STU are not joined, check via codegen var b = beginCell(); @@ -267,6 +278,7 @@ fun test20() { return (s.loadBool(), s.loadBool(), s.loadBool(), s.loadUint(8), s.loadBool(), s.loadCoins()); } +@method_id(121) fun test21(s: slice) { // successive skipBits are also joined var x = 8; @@ -433,6 +445,30 @@ fun test34(p: int, n: int) { return b; } +@method_id(135) +fun test35(overflowMode: int) { + try { + val b: builder = match (overflowMode) { + 1 => triggerOverflowIntConst(), + 2 => triggerOverflowUintConst(), + 3 => beginCell().storeUint(10, -4), + 4 => beginCell().storeInt(100, 1), + 5 => beginCell().storeInt(1, 0), + 6 => beginCell().storeInt(115792089237316195423570985008687907853269984665640564039457584007913129639935, 256), + 7 => beginCell().storeInt(15, 4), + 8 => beginCell().storeUint(1<<170, 169), + else => beginCell(), + }; + return b.endCell().beginParse().loadUint(1) * 1000 + } + catch (ex) { return ex } +} + +@method_id(136) +fun test36() { + return beginCell().storeInt(0, 0) +} + fun main(): int { return 0; } @@ -467,6 +503,15 @@ fun main(): int { @testcase | 132 | 0 | BC{00080000000a} @testcase | 133 | 0 | BC{00020a} @testcase | 134 | 0 8 | BC{00020a} +@testcase | 135 | 1 | 5 +@testcase | 135 | 2 | 5 +@testcase | 135 | 3 | 5 +@testcase | 135 | 4 | 5 +@testcase | 135 | 5 | 5 +@testcase | 135 | 6 | 5 +@testcase | 135 | 7 | 5 +@testcase | 135 | 8 | 5 +@testcase | 136 | | BC{0000} We test that consequtive storeInt/storeUint with constants are joined into a single number @@ -628,4 +673,32 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin }> """ +@fif_codegen +""" + triggerOverflowIntConst() PROC:<{ + 10 PUSHINT + NEWC + 4 STI + }> +""" + +@fif_codegen +""" + triggerOverflowUintConst() PROC:<{ + 123 PUSHINT + NEWC + 6 STU + }> +""" + +@fif_codegen +""" + test36() PROC:<{ + 0 PUSHINT + NEWC + OVER + STIX + }> +""" + */ diff --git a/tolk-tester/tests/comments-tests.tolk b/tolk-tester/tests/comments-tests.tolk index 96caefe92..828780806 100644 --- a/tolk-tester/tests/comments-tests.tolk +++ b/tolk-tester/tests/comments-tests.tolk @@ -18,6 +18,7 @@ fun demo_fields_def(x: int) { } fun main() { + var cb = demo_fields_def; return 1; } diff --git a/tolk-tester/tests/dicts-demo.tolk b/tolk-tester/tests/dicts-demo.tolk index f53debb7f..990d57a86 100644 --- a/tolk-tester/tests/dicts-demo.tolk +++ b/tolk-tester/tests/dicts-demo.tolk @@ -104,6 +104,26 @@ fun test105(takeNext: bool) { return data!.loadUint(8); } +@method_id(106) +fun test106() { + var dict = createEmptyDict(); + dict.addIntToIDict(2, 102); + dict.addIntToIDict(-1, 101); + dict.addIntToIDict(-4, 104); + dict.addIntToIDict(3, 103); + + var m = createMapFromLowLevelDict(dict); + val fk = m.findFirst().getKey(); + val fv = m.findFirst().loadValue(); + var t = createEmptyTuple(); + var r = m.findKeyLess(3); + while (r.isFound) { + t.push(r.loadValue()); + r = m.iteratePrev(r); + } + return (fk, fv, t); +} + fun main() {} /** @@ -115,4 +135,5 @@ fun main() {} @testcase | 104 | | 12 34 56 78 0 @testcase | 105 | -1 | 1 @testcase | 105 | 0 | 2 +@testcase | 106 | | -4 104 [ 102 101 104 ] */ diff --git a/tolk-tester/tests/enums-tests.tolk b/tolk-tester/tests/enums-tests.tolk new file mode 100644 index 000000000..c61faada2 --- /dev/null +++ b/tolk-tester/tests/enums-tests.tolk @@ -0,0 +1,307 @@ +const SIX = 6 + +enum EEmpty {} + +enum Color { + Red, + Green, + Blue +} + +type ColorAlias = Color + +struct WithColor { + c1: Color + c2: Color = Color.Green +} + +enum EPartialAssigned { + M100 = -100, + M99, + P1 = (1 + 0) << 0; + P2 = ((+1 + 1) << 1) / 2; + P3 + P4 + P5 = 0b101, + P6 = SIX; + Max = 0x7FFFFFFF, +} + +enum Err { + TooLowId = 0x100, + TooHighId, + DisabledId, +} + +struct Box { + value: T +} + +fun checkIsBlue(c: Color) { + return c == Color.Blue +} + +fun Color.isBlue(self) { + return self == Color.Blue +} + +fun Color.createBlue() { + return Color.Blue +} + +const C_BLUE = Color.Blue +const C_TEN = 5 * (Color.Green as int) * 2 + +// these methods do not bother Color.Red and other +fun T.Red() { return 123 } +fun T.Max() { return -C_TEN } +fun int.Max(self) { return self } +fun Color.Max(): EPartialAssigned { return ((EPartialAssigned.Max as int) + 1) as EPartialAssigned } + +@noinline +fun checkIsColor(obj: Color | T) { return obj is Color } + +@inline_ref +fun Color.mapToNumber(self) { + // match over enum is exhaustive, it's like "if Red {} else if Green {} else {}" + return match (self) { + Color.Red => 100, + Color.Green => 200, + Color.Blue => 300, // this works like `else` and handles invalid values at runtime also + } +} + + +@method_id(101) +fun test1() { + var c = Color.Red; + + return ( + checkIsBlue(Color.Blue), checkIsBlue(c), checkIsBlue(Color.createBlue()), + Color.Blue.isBlue(), Color.Red.isBlue(), Color.createBlue()!!.isBlue(), C_BLUE.isBlue(), + Color.createBlue() + ); +} + +@method_id(102) +fun test2(c: Color) { + return (c is Color, c !is Color, c is EPartialAssigned, c is EEmpty, c is int, c is int8); +} + +@method_id(103) +fun test3() { + assert (EPartialAssigned.P4 as int == 4) throw 123; + assert (EPartialAssigned.M99 as int <= 0) throw 123; + assert (EPartialAssigned.P3 == (3 as EPartialAssigned)) throw 123; + assert (EPartialAssigned.P3 != (4 as EPartialAssigned)) throw 123; + return ( + EPartialAssigned.M100, + EPartialAssigned.M99, + EPartialAssigned.P1, + EPartialAssigned.P2, + EPartialAssigned.P3, + EPartialAssigned.P4, + EPartialAssigned.P5, + EPartialAssigned.P6, + EPartialAssigned.Max, + ) +} + +@method_id(104) +fun test4(chooseGreen: bool) { + var c: ColorAlias = chooseGreen ? Color.Green : ColorAlias.Blue; + return (c is Color, (c!) is ColorAlias, ColorAlias.Blue, ColorAlias.createBlue().isBlue(), c); +} + +@method_id(105) +fun test5() { + return (int.Red(), int8.Max(), 456.Max(), EPartialAssigned.Max, Color.Max()); +} + +@method_id(106) +fun test6(c: Color) { + val p = c as EPartialAssigned; + val e = c as EEmpty; + match (e) { EEmpty => {} } + return (p, e, c as int, c as int?, 100 as Color, -100 as EEmpty); +} + +@method_id(107) +fun test7() { + return ( + WithColor{c1: Color.Red}, + WithColor{c1: -100 as Color, c2: Color.createBlue()} + ) +} + +@method_id(108) +fun test8() { + var (c0: Color | int, c1: Color | int) = (Color.Red, 123); + var (u1, u2) = (Color.Red as Color | int, 0 as Color | int); + var (b1, b2, b3) = (Color.Red as Color | EPartialAssigned | builder, EPartialAssigned.Max as ColorAlias | builder | EPartialAssigned, beginCell() as builder | Color | EPartialAssigned); + return ( + checkIsColor(u1), checkIsColor(u2), + checkIsColor(b1), checkIsColor(b2), checkIsColor(b3), + b2 is EPartialAssigned, + u1, + match (u1) { int => -100, Color => -200 } + ) +} + +@method_id(109) +fun test9() { + return ( + Color.Red.mapToNumber(), Color.Green.mapToNumber(), Color.Blue.mapToNumber(), + // this is UB, the compiler may do any optimizations + (100 as Color).mapToNumber(), (-99 as Color).mapToNumber(), + ); +} + +@method_id(110) +fun test10(c: Color) { + var t = createEmptyTuple(); + match (c) { + Color.Red => { t.push(1) } + else => { t.push(-100) } + } + match (c) { + Color.Green => { t.push(2) } + Color.Blue => { t.push(3) } + Color.Red => { t.push(1) } + } + match (c is Color) { + true => { t.push(-100) } + } + match (c) { + Color => { t.push(88) } + } + match (c) { + Color.Blue => { t.push(3) } + Color.Green => { t.push(2) } + else => { t.push(-100) } + } + match (c) { + Color.Red => { t.push(1) } + Color.Blue => { t.push(3) } + Color.Green => { t.push(2) } + } + return t; +} + +@method_id(111) +fun test11(b1Value: Color) { + var b1: Box = { value: b1Value }; + var b2 = Box { value: Color.Green }; + var b3 = Box { value: Color.Blue }; + return (b1.value == b2.value, b1.value != b2.value, b1.value == b3.value, b1.value != b3.value) +} + +@method_id(112) +fun test12(c1: Color, c2: Color) { + var t = createEmptyTuple(); + + match (c1) { + Color.Red => { t.push(10 + c2.mapToNumber()) } + Color.Green => { t.push(20 + c2.mapToNumber()) } + Color.Blue => { t.push(30 + c2.mapToNumber()) } + } + + match (c2) { + Color.Red => { + match (c1) { + Color.Red => t.push(100), + Color.Blue => t.push(300), + Color.Green => t.push(200), + } + } + Color.Blue => match (c1) { + Color.Green => t.push(210), + Color.Red => t.push(110), + Color.Blue => t.push(310), + }, + Color.Green => match (c1) { + Color.Green => t.push(220), + Color.Red => t.push(120), + Color.Blue => t.push(320), + } + } + + t.push((999 as Color).mapToNumber()); + return t; +} + +@method_id(113) +fun test13(x: int) { + try { + if (x < 0) { throw Err.TooLowId } + assert (x < 100) throw Err.TooHighId; + assert (x != 10, Err.DisabledId); + return (0, false); + } catch (excno) { + return (excno, excno as Err == Err.TooLowId); + } +} + + +fun main(c: Color) { + __expect_type(c, "Color"); + __expect_type(C_BLUE, "Color"); + __expect_type(Color.createBlue(), "Color"); + __expect_type(ColorAlias.Red, "Color"); + __expect_type(EPartialAssigned.Max, "EPartialAssigned"); + __expect_type(Box{value:Color.Red}, "Box"); + return c; +} + + +/** +@testcase | 0 | 123 | 123 +@testcase | 101 | | -1 0 -1 -1 0 -1 -1 2 +@testcase | 102 | 1 | -1 0 0 0 0 0 +@testcase | 103 | | -100 -99 1 2 3 4 5 6 2147483647 +@testcase | 104 | -1 | -1 -1 2 -1 1 +@testcase | 105 | | 123 -10 456 2147483647 2147483648 +@testcase | 106 | 1 | 1 1 1 1 100 -100 +@testcase | 107 | | 0 1 -100 2 +@testcase | 108 | | -1 0 -1 0 0 -1 0 typeid-1 -200 +@testcase | 109 | | 100 200 300 300 300 +@testcase | 110 | 0 | [ 1 1 -100 88 -100 1 ] +@testcase | 110 | 1 | [ -100 2 -100 88 2 2 ] +@testcase | 110 | 2 | [ -100 3 -100 88 3 3 ] +@testcase | 110 | 44 | [ -100 1 -100 88 -100 2 ] +@testcase | 111 | 2 | 0 -1 -1 0 +@testcase | 112 | 0 1 | [ 210 120 300 ] +@testcase | 112 | 2 4 | [ 330 320 300 ] +@testcase | 113 | -8 | 256 -1 +@testcase | 113 | 111 | 257 0 +@testcase | 113 | 10 | 258 0 +@testcase | 113 | 50 | 0 0 + +@fif_codegen +""" + test1() PROC:<{ + TRUE + FALSE + TRUE + TRUE + FALSE + TRUE + TRUE + 2 PUSHINT + }> +""" + +@fif_codegen +""" + test2() PROC:<{ // c + DROP // + -1 PUSHINT // '1=-1 + 0 PUSHINT // '1=-1 '2 + 0 PUSHINT // '1=-1 '2 '3=0 + s0 s0 s0 PUSH3 // '1=-1 '2 '3=0 '4=0 '5=0 '6=0 + }> +""" + +@fif_codegen DECLPROC checkIsColor() + +*/ diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index b92f27a6e..d44f083f8 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -15,7 +15,7 @@ fun tuplePush(mutate t: tuple, value: T): void @method_id(101) fun test101(x: int) { var (a, b, c) = (x, (x,x), [x,x]); - return (eq1(a), eq1(b), eq1(c), eq2(a), eq2(b), eq2(c), eq3(a), eq4(b), eq3(createEmptyTuple())); + return (eq1(a), eq1(b), eq1(c), eq2(a), eq2(b), eq2(c), eq3(a), eq4(b), eq3(createEmptyTuple()), eq1(Color.Red)); } fun getTwo(): X { return 2 as X; } @@ -28,6 +28,7 @@ struct Wrapper { struct Err { errPayload: T } +enum Color { Red, Green, Blue } @method_id(102) fun test102(): (int, int, int, [int, int], Wrapper< Wrapper >) { @@ -211,6 +212,7 @@ fun test113(a: int | slice) { fun main(x: int): (int, [Tup2Int]) { __expect_type(test110, "() -> (Tensor2Int, Tensor2Int)"); + __expect_type(eq2(10 as Color), "Color"); try { if(x) { throw (1, x); } } catch (excNo, arg) { return (arg as int, [[eq2(arg as int), getTwo()]]); } @@ -219,7 +221,7 @@ fun main(x: int): (int, [Tup2Int]) { /** @testcase | 0 | 1 | 1 [ [ 1 2 ] ] -@testcase | 101 | 0 | 0 0 0 [ 0 0 ] 0 0 0 [ 0 0 ] 0 0 0 [] +@testcase | 101 | 0 | 0 0 0 [ 0 0 ] 0 0 0 [ 0 0 ] 0 0 0 [] 0 @testcase | 102 | | 2 2 2 [ 2 2 ] 2 @testcase | 103 | 0 | 0 100 100 @testcase | 104 | 0 | [ 1 (null) 2 ] [ 2 -1 0 ] @@ -241,6 +243,7 @@ fun main(x: int): (int, [Tup2Int]) { @fif_codegen DECLPROC eq1() @fif_codegen DECLPROC eq1() @fif_codegen DECLPROC eq1>>() +@fif_codegen DECLPROC eq1() // was inlined @fif_codegen_avoid DECLPROC getTwo() diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index 2a6684f0d..82683af83 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -34,11 +34,23 @@ fun wrap2(value: T) { return WrapperAlias { value }; } +fun wrap0(): Wrapper { + return { value: 0 } +} + +fun wrapNull(): Wrapper { + return { value: null } +} + fun makePair(a: A, b: B) { var c: Pair = { first: a, second: b }; return c; } +fun makePairOf0(): Pair { + return { first: 0, second: 0 } +} + @method_id(101) fun test1() { @@ -330,8 +342,8 @@ type StrangeAlsoInt = StrangeInt<()>; fun test19() { var i1: StrangeInt = 20; var i2: StrangeInt = 30; - var i3: StrangeInt> = 40 as StrangeAlsoInt; - i1 = i2; i2 = i1; i1 = i3; i3 = i1; + var i3: StrangeInt> = 40; + i1 = i2 as StrangeInt; i1 = i3 as StrangeInt; return (i1 as StrangeInt<()>, i3 as StrangeAlsoInt, i1 is StrangeInt, i1 is StrangeInt, i3 is StrangeInt, i3 is StrangeAlsoInt); } @@ -376,7 +388,7 @@ fun test22(w: int | WrapperAlias | slice) { WrapperAlias => {} slice => {} } - __expect_type(w, "int | Wrapper | slice"); + __expect_type(w, "int | WrapperAlias | slice"); match (w) { MyInt => {} slice => {} @@ -428,6 +440,72 @@ fun test25() { } } +@method_id(126) +fun test26() { + var m1: Wrapper? = wrap0(); + var m2 = wrap0() as Wrapper?; + var m3: WrapperAlias? = wrap0(); + var m4: Wrapper? = wrapNull(); + var m5 = wrap0() as Wrapper?; + var m6 = wrapNull() as WrapperAlias?; + var m7 = null as Wrapper?; + __expect_type(m1, "Wrapper"); + __expect_type(m2, "Wrapper?"); + __expect_type(m3, "WrapperAlias"); + __expect_type(m4, "Wrapper"); + return (m1, m2, m3.value, m4.value, 777, m5, m6, m7); +} + +fun test27() { + var pp1: Pair | int = makePairOf0(); + var pp2 = makePairOf0() as Pair | Wrapper; + __expect_type(pp1, "Pair"); + __expect_type(pp2, "Pair | Wrapper"); +} + +struct Box { + private readonly item: T +} + +fun Box.create(v: T) { + return Box { item: v } +} + +fun Box.itemEquals(self, cmp: T) { + return self.item == cmp +} + +fun Box.itemEqualsTo(self, cmp: Box) { + return self.item == cmp.item +} + +fun Box.clone(self) { + var b = Box { item: self.item as U }; + b.item; + return b; +} + +fun Box.areEqual(b1: Box, b2: Box) { + return b1.item == b2.item +} + +fun Box.specifiedGetItem(self) { + return self.item +} + +@method_id(128) +fun test28() { + var b1 = Box.create(5); + var b2 = Box.create(4); + var b3 = b2.clone(); + __expect_type(b3, "Box"); + return ( + b1.itemEquals(0), b2.itemEquals(4), + b1.itemEqualsTo(b2), b2.itemEqualsTo(b3), + Box.areEqual(b1, b1), + ); +} + fun main(c: Wrapper, d: WrappedInt) { __expect_type(c, "Wrapper"); @@ -453,15 +531,15 @@ fun main(c: Wrapper, d: WrappedInt) { @testcase | 104 | | 11 11 13 13 13 2 @testcase | 105 | 0 | 80 @testcase | 105 | -1 | (null) -@testcase | 106 | | (null) (null) 0 777 1 2 typeid-14 +@testcase | 106 | | (null) (null) 0 777 1 2 typeid-15 @testcase | 107 | 5 | 20 20 5 5 20 777 (null) (null) (null) 0 -@testcase | 108 | 0 | 777 typeid-15 777 typeid-15 +@testcase | 108 | 0 | 777 typeid-1 777 typeid-1 @testcase | 108 | -1 | 777 0 777 0 @testcase | 109 | | 40 40 70 @testcase | 110 | | 20 1 20 42 -@testcase | 111 | | 5 1 typeid-4 5 1 typeid-4 5 1 typeid-4 5 1 typeid-4 -@testcase | 112 | -1 | 10 1 777 10 1 typeid-5 -@testcase | 112 | 0 | 20 1 777 (null) 0 typeid-6 +@testcase | 111 | | 5 1 typeid-2 5 1 typeid-2 5 1 typeid-2 5 1 typeid-2 +@testcase | 112 | -1 | 10 1 777 10 1 typeid-3 +@testcase | 112 | 0 | 20 1 777 (null) 0 typeid-4 @testcase | 113 | | 30 -1 @testcase | 114 | | 999 (null) 2 @testcase | 115 | | 10 0 200 1 -1 200 1 -1 @@ -469,8 +547,10 @@ fun main(c: Wrapper, d: WrappedInt) { @testcase | 117 | | 100 123 @testcase | 118 | | 123 1 777 123 1 -1 2 @testcase | 119 | | 40 40 -1 -1 -1 -1 -@testcase | 120 | | (null) typeid-9 777 (null) (null) 0 typeid-11 -1 -1 +@testcase | 120 | | (null) typeid-9 777 (null) (null) 0 typeid-10 -1 -1 @testcase | 123 | | 10 10 10 10 10 +@testcase | 126 | | 0 0 0 (null) 777 0 typeid-12 (null) typeid-12 (null) 0 +@testcase | 128 | | 0 -1 0 -1 -1 @fif_codegen diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index c5f3e038c..6833d3867 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -63,6 +63,7 @@ fun mySend(p: Parameters): int { return total; } +@method_id(101) fun test1() { eqUnusedT(100); eqUnusedU(100); @@ -175,7 +176,7 @@ fun main() { /** @testcase | 103 | | 10 20 30 777 40 40 @testcase | 104 | | (null) (null) (null) -@testcase | 105 | | -1 (null) 0 (null) typeid-6 777 0 123 0 456 typeid-5 +@testcase | 105 | | -1 (null) 0 (null) typeid-2 777 0 123 0 456 typeid-1 @testcase | 106 | | -1 0 -1 -1 @testcase | 107 | | 1 10 110 117 @testcase | 108 | | 10 (null) 20 diff --git a/tolk-tester/tests/handle-msg-6.tolk b/tolk-tester/tests/handle-msg-6.tolk index 0d349f9cc..57668f2bf 100644 --- a/tolk-tester/tests/handle-msg-6.tolk +++ b/tolk-tester/tests/handle-msg-6.tolk @@ -5,7 +5,7 @@ fun test1() { } fun onBouncedMessage(in: InMessageBounced) { - if (in.valueExtra != null) { + if (!in.valueExtra.isEmpty()) { return; } throw in.originalForwardFee; @@ -23,7 +23,7 @@ fun onInternalMessage(in: InMessage) { onBouncedMessage() PROC:<{ DROP INMSG_VALUEEXTRA - ISNULL + DICTEMPTY IFNOTJMP:<{ }> INMSG_FWDFEE diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index af6888df7..c30911015 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -179,6 +179,7 @@ fun getConstTuple(): Tup2Int { return [1,2]; } +@noinline fun testCodegenNoPureIndexedAccess() { (getConstTuple().1, getConstTuple().0) = (3, 4); return 0; @@ -277,7 +278,9 @@ fun test123() { return t; } -fun main(){} +fun main(){ + testCodegenNoPureIndexedAccess(); +} /** diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index 9c836baf4..7aa5a3cb3 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -75,7 +75,7 @@ fun test5(x: int) { __expect_type([x, x >= 1], "[int, bool]"); __expect_type([x, x >= 1, null as slice?], "[int, bool, slice?]"); __expect_type((x, [x], [[x], x]), "(int, [int], [[int], int])"); - __expect_type(contract.getOriginalBalanceWithExtraCurrencies(), "[coins, dict]"); + __expect_type(contract.getOriginalBalanceWithExtraCurrencies(), "[coins, ExtraCurrenciesMap]"); } fun test6() { @@ -111,8 +111,8 @@ fun alwaysThrowsNotAnnotated2() { alwaysThrows(); } fun test9() { __expect_type(alwaysThrows(), "never"); __expect_type(alwaysThrows, "() -> never"); - __expect_type(alwaysThrowsNotAnnotated(), "void"); - __expect_type(alwaysThrowsNotAnnotated2(), "void"); + __expect_type(alwaysThrowsNotAnnotated(), "never"); + __expect_type(alwaysThrowsNotAnnotated2(), "never"); } diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk index d85c5543b..d27e79bb8 100644 --- a/tolk-tester/tests/inline-tests.tolk +++ b/tolk-tester/tests/inline-tests.tolk @@ -537,7 +537,7 @@ fun usedIn10ButDeclaredBelow(x: int) { """ wrap16() PROC:<{ IFJMP:<{ - 139 PUSHINT + 129 PUSHINT OVER EQUAL IFJMP:<{ @@ -545,7 +545,7 @@ fun usedIn10ButDeclaredBelow(x: int) { 0 GTINT 100 THROWIFNOT }> - 140 PUSHINT + 130 PUSHINT s2 POP EQUAL IFJMP:<{ diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index cd557092e..2555d983c 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -201,6 +201,7 @@ fun test11() { ]; } +@method_id(112) fun test12() { var a = ton("1") + ton("2"); // check absence in fif codegen return ton("0.1"); @@ -247,7 +248,7 @@ fun main() { @testcase | 110 | | 50000000 50000100 1234000000 51000000 @testcase | 111 | | [ 1000000000 1000000000 1000000000 -321123456789 321123456789 1100000000 ] @testcase | 114 | 5 | (null) (null) 0 -@testcase | 114 | 15 | 15 2 typeid-2 +@testcase | 114 | 15 | 15 2 typeid-1 @fif_codegen DECLPROC assign0() @fif_codegen DECLPROC assign0() diff --git a/tolk-tester/tests/invalid-declaration/err-1039.tolk b/tolk-tester/tests/invalid-declaration/err-1039.tolk new file mode 100644 index 000000000..52143ab75 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1039.tolk @@ -0,0 +1,7 @@ + +enum Color { Red = Color.Red } + +/** +@compilation_should_fail +@stderr referencing other members in initializers is not allowed yet + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1480.tolk b/tolk-tester/tests/invalid-declaration/err-1480.tolk new file mode 100644 index 000000000..667c5af2b --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1480.tolk @@ -0,0 +1,8 @@ + +enum C3 : int { +} + +/** +@compilation_should_fail +@stderr serialization type of `enum` must be intN: `int8` / `uint32` / etc. + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1697.tolk b/tolk-tester/tests/invalid-declaration/err-1697.tolk new file mode 100644 index 000000000..cc8a90432 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1697.tolk @@ -0,0 +1,7 @@ + +enum Color { Red = ton("0.05") as int } + +/** +@compilation_should_fail +@stderr unsupported operation in `enum` member value + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1700.tolk b/tolk-tester/tests/invalid-declaration/err-1700.tolk new file mode 100644 index 000000000..018a583fe --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1700.tolk @@ -0,0 +1,11 @@ +enum Over { + MaxIntMinus1 = +115792089237316195423570985008687907853269984665640564039457584007913129639935 - 1, + MaxInt, + MaxIntPlus1, +} + +/** +@compilation_should_fail +@stderr integer overflow +@stderr MaxIntPlus1 + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1709.tolk b/tolk-tester/tests/invalid-declaration/err-1709.tolk new file mode 100644 index 000000000..cb3849e6c --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1709.tolk @@ -0,0 +1,8 @@ +global g: map + +/** +@compilation_should_fail +@stderr invalid `map`: type `coins` can not be used as a key +@stderr because its binary size is not constant: it's 4..124 bits +@stderr global g + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1710.tolk b/tolk-tester/tests/invalid-declaration/err-1710.tolk new file mode 100644 index 000000000..5235b9441 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1710.tolk @@ -0,0 +1,14 @@ +struct WithRef { + v: int32; + r: cell?; +} + +fun main(s: map) { + +} + +/** +@compilation_should_fail +@stderr invalid `map`: type `WithRef` can not be used as a key +@stderr because it may contain a cell reference, not only data bits + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1711.tolk b/tolk-tester/tests/invalid-declaration/err-1711.tolk new file mode 100644 index 000000000..f00d618ff --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1711.tolk @@ -0,0 +1,13 @@ +type ForwardPayload = RemainingBitsAndRefs + +struct WithPayload { + w: map +} + +/** +@compilation_should_fail +@stderr invalid `map`: type `ForwardPayload` can not be used as a key +@stderr because it does not specify keyLen for a dictionary +@stderr hint: use `address` if a key is an internal address +@stderr hint: use `bits128` and similar if a key represents fixed-width data + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1806.tolk b/tolk-tester/tests/invalid-declaration/err-1806.tolk new file mode 100644 index 000000000..240d1267c --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1806.tolk @@ -0,0 +1,12 @@ + +fun main() { + var m: map; +} + +/** +@compilation_should_fail +@stderr invalid `map`: type `int` can not be used as a value +@stderr because it can not be serialized +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1839.tolk b/tolk-tester/tests/invalid-declaration/err-1839.tolk new file mode 100644 index 000000000..595a853b5 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1839.tolk @@ -0,0 +1,8 @@ +enum Over { + A = (+115792089237316195423570985008687907853269984665640564039457584007913129639934 << 1) * 2, +} + +/** +@compilation_should_fail +@stderr integer overflow + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1945.tolk b/tolk-tester/tests/invalid-declaration/err-1945.tolk new file mode 100644 index 000000000..a1d825a4c --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1945.tolk @@ -0,0 +1,22 @@ +type MInt = int +struct WI { v: MInt } + +struct P { + w: map +} + +fun main() { + +} + +/** +@compilation_should_fail +@stderr invalid `map`: type `(bool, WI)` can not be used as a value +@stderr because it can not be serialized +@stderr because element `tensor.1` of type `WI` can't be serialized +@stderr because field `WI.v` of type `MInt` can't be serialized +@stderr because alias `MInt` expands to `int` +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. +@stderr w: map + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4081.tolk b/tolk-tester/tests/invalid-semantics/err-4081.tolk index 4442cf2d5..7a3d9bcfb 100644 --- a/tolk-tester/tests/invalid-semantics/err-4081.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4081.tolk @@ -15,6 +15,6 @@ fun main(w: int | WrapperAlias | WrapperAlias) { /** @compilation_should_fail -@stderr wrong pattern matching: duplicated `Wrapper` +@stderr wrong pattern matching: duplicated `WrapperAlias` @stderr WrapperAlias */ diff --git a/tolk-tester/tests/invalid-semantics/err-4104.tolk b/tolk-tester/tests/invalid-semantics/err-4104.tolk index 3ca7ef1bc..7cb8c8ef9 100644 --- a/tolk-tester/tests/invalid-semantics/err-4104.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4104.tolk @@ -8,7 +8,7 @@ fun Container.createFrom(item: U): Container { fun cantUseAsNonCall() { Container.createFrom; // ok - Container.createFrom>; // ok + Container.createFrom>; // ok Container.createFrom; } diff --git a/tolk-tester/tests/invalid-semantics/err-4109.tolk b/tolk-tester/tests/invalid-semantics/err-4109.tolk new file mode 100644 index 000000000..9f4a5771b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4109.tolk @@ -0,0 +1,17 @@ +struct Wrapper { + value: T; +} + +fun wrap0(): Wrapper { + return { value: 0 } +} + +fun main() { + var m: Wrapper | Wrapper = wrap0(); +} + +/** +@compilation_should_fail +@stderr can not deduce T for generic function `wrap0` +@stderr instantiate it manually with `wrap0<...>()` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4130.tolk b/tolk-tester/tests/invalid-semantics/err-4130.tolk new file mode 100644 index 000000000..907727f98 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4130.tolk @@ -0,0 +1,22 @@ +struct Point { + x: int + y: int +} + +struct ROLine { + readonly p1: Point + readonly p2: Point +} + +fun eq(v: T) { return v } + +fun f(l: ROLine) { + l.p1; l.p2; l.p1.x; eq(l.p2.y); eq(eq(l).p2).y; + + l.p2!.x += 1; +} + +/** +@compilation_should_fail +@stderr modifying readonly field `ROLine.p2` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4131.tolk b/tolk-tester/tests/invalid-semantics/err-4131.tolk new file mode 100644 index 000000000..c3403b3b4 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4131.tolk @@ -0,0 +1,12 @@ +struct ImmSlice { + readonly s: slice +} + +fun ImmSlice.read(mutate self) { + self.s.loadRef() +} + +/** +@compilation_should_fail +@stderr modifying readonly field `ImmSlice.s` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4132.tolk b/tolk-tester/tests/invalid-semantics/err-4132.tolk new file mode 100644 index 000000000..6de7eb560 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4132.tolk @@ -0,0 +1,15 @@ +struct ROPoint { + readonly x: int + readonly y: int +} + +fun increment(mutate x: int) {} + +fun main(p1: ROPoint, p2: ROPoint) { + increment(mutate (p1 = p2).x); +} + +/** +@compilation_should_fail +@stderr modifying readonly field `ROPoint.x` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4318.tolk b/tolk-tester/tests/invalid-semantics/err-4318.tolk new file mode 100644 index 000000000..9809b7c8f --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4318.tolk @@ -0,0 +1,13 @@ +struct Point { + x: int + y: int +} + +fun Point.getX() { + return self.x +} + +/** +@compilation_should_fail +@stderr using `self` in a static method + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4359.tolk b/tolk-tester/tests/invalid-semantics/err-4359.tolk new file mode 100644 index 000000000..4b487e2b7 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4359.tolk @@ -0,0 +1,9 @@ +fun m() { + var c = (12 as int64).toCell(); + var s = c.tvmCell.beginParse(); +} + +/** +@compilation_should_fail +@stderr field `Cell.tvmCell` is private + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4395.tolk b/tolk-tester/tests/invalid-semantics/err-4395.tolk new file mode 100644 index 000000000..c258ff379 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4395.tolk @@ -0,0 +1,10 @@ +enum Color { Red, Green, Blue, } + +fun main() { + Color.Red = 123 as Color; +} + +/** +@compilation_should_fail +@stderr modifying immutable constant + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4414.tolk b/tolk-tester/tests/invalid-semantics/err-4414.tolk new file mode 100644 index 000000000..481ddcc3a --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4414.tolk @@ -0,0 +1,11 @@ +struct Point { private x: int, private y: int } +struct Point3d { x: int, y: int, z: int } + +fun Point3d.create() { + return Point { x: 10, y: 20 } +} + +/** +@compilation_should_fail +@stderr field `Point.x` is private + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4415.tolk b/tolk-tester/tests/invalid-semantics/err-4415.tolk new file mode 100644 index 000000000..db3faf01d --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4415.tolk @@ -0,0 +1,11 @@ +struct Wrap { private item: T } + +fun m(w: Wrap) { + w = w; + return w.item!; +} + +/** +@compilation_should_fail +@stderr field `Wrap.item` is private + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4518.tolk b/tolk-tester/tests/invalid-semantics/err-4518.tolk new file mode 100644 index 000000000..7fbf9e01a --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4518.tolk @@ -0,0 +1,13 @@ +struct Point { x: int, y: int } +fun Point.create() {} + +fun nnn() {} + +fun main() { + Point.create = nnn; +} + +/** +@compilation_should_fail +@stderr invalid left side of assignment + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4539.tolk b/tolk-tester/tests/invalid-semantics/err-4539.tolk new file mode 100644 index 000000000..cc15288ee --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4539.tolk @@ -0,0 +1,25 @@ +import "@stdlib/tvm-dicts" + +type BalanceList = dict +type AssetList = dict + +fun BalanceList.validate(self): int1 { return 0 } +fun AssetList.validate(self): int2 { return 0 } + +fun f(b: BalanceList) { + if (b == null) { return } + else { b = createEmptyDict() } + __expect_type(b, "BalanceList"); + + if (b == null) { return } + __expect_type(b, "cell"); + + b.validate() +} + +/** +@compilation_should_fail +@stderr call to method `validate` for type `cell` is ambiguous +@stderr candidate function: `BalanceList.validate` +@stderr candidate function: `AssetList.validate` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4560.tolk b/tolk-tester/tests/invalid-semantics/err-4560.tolk new file mode 100644 index 000000000..507b9ccc2 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4560.tolk @@ -0,0 +1,17 @@ +struct Wrapper { item: T } + +type AssetBuilder = builder +type BalanceBuilder = builder + +fun Wrapper.validate(self) { return 0 } + +fun main() { + var b: BalanceBuilder = beginCell(); + Wrapper{item:b}.validate(); +} + +/** +@compilation_should_fail +@stderr method `validate` not found for type `Wrapper` +@stderr but it exists for type `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4572.tolk b/tolk-tester/tests/invalid-semantics/err-4572.tolk index a62b63a0f..694f6d704 100644 --- a/tolk-tester/tests/invalid-semantics/err-4572.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4572.tolk @@ -1,6 +1,8 @@ +fun eq(v: T) { return v } + fun checkCantMutateFieldOfImmutableTuple() { val ks = null as [int, [int, [int]]]?; - ks!.1.1.0 = 10; + eq(ks!.1.1 = [10]).0 = 10; } /** diff --git a/tolk-tester/tests/invalid-semantics/err-4619.tolk b/tolk-tester/tests/invalid-semantics/err-4619.tolk new file mode 100644 index 000000000..e62cc22b5 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4619.tolk @@ -0,0 +1,15 @@ +struct Inc { v: int32 } + +type Msg = Inc + +fun main(s: slice) { + val msg = lazy Msg.fromSlice(s); + match (msg) { + else => throw 0xFFFF + } +} + +/** +@compilation_should_fail +@stderr `match` contains only `else`, but no variants + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4680.tolk b/tolk-tester/tests/invalid-semantics/err-4680.tolk new file mode 100644 index 000000000..175ec6b65 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4680.tolk @@ -0,0 +1,15 @@ +type BalanceDict = dict + +fun dict.doSome(self) {} +fun BalanceDict.doSome(self) {} + +fun m(d: dict) { + d.doSome() +} + +/** +@compilation_should_fail +@stderr call to method `doSome` for type `dict` is ambiguous +@stderr candidate function: `dict.doSome` +@stderr candidate function: `BalanceDict.doSome` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4681.tolk b/tolk-tester/tests/invalid-semantics/err-4681.tolk new file mode 100644 index 000000000..0974c50dd --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4681.tolk @@ -0,0 +1,15 @@ +type BalanceDict = dict + +fun dict.doSome(self) {} +fun BalanceDict.doSome(self) {} + +fun m(d: BalanceDict) { + d.doSome() +} + +/** +@compilation_should_fail +@stderr call to method `doSome` for type `BalanceDict` is ambiguous +@stderr candidate function: `dict.doSome` +@stderr candidate function: `BalanceDict.doSome` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4810.tolk b/tolk-tester/tests/invalid-semantics/err-4810.tolk new file mode 100644 index 000000000..7c13cf87b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4810.tolk @@ -0,0 +1,10 @@ +enum Color { Red, Green, Blue } + +fun main() { + return Color.Red +} + +/** +@compilation_should_fail +@stderr type arguments not expected here + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4822.tolk b/tolk-tester/tests/invalid-semantics/err-4822.tolk index 85cf4073b..108153c0c 100644 --- a/tolk-tester/tests/invalid-semantics/err-4822.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4822.tolk @@ -8,8 +8,8 @@ struct Coord2 { p2: Point; } -fun increment(mutate x: int) { - x += 1; +fun int.increment(mutate self) { + self += 1; } fun checkCantMutateFieldOfImmutableTuple(p1: Point?, p2: Point) { @@ -19,5 +19,5 @@ fun checkCantMutateFieldOfImmutableTuple(p1: Point?, p2: Point) { /** @compilation_should_fail -@modifying immutable variable `c2` +@stderr modifying immutable variable `c2` */ diff --git a/tolk-tester/tests/invalid-semantics/err-4900.tolk b/tolk-tester/tests/invalid-semantics/err-4900.tolk new file mode 100644 index 000000000..292252931 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4900.tolk @@ -0,0 +1,13 @@ +fun int.realValue(self): int8 { return 1 } +fun int?.realValue(self): uint256 { return 1 } + +fun main() { + (5 as int8).realValue(); +} + +/** +@compilation_should_fail +@stderr call to method `realValue` for type `int8` is ambiguous +@stderr candidate function: `int.realValue` +@stderr candidate function: `int?.realValue` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4901.tolk b/tolk-tester/tests/invalid-semantics/err-4901.tolk new file mode 100644 index 000000000..a21db6fee --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4901.tolk @@ -0,0 +1,13 @@ +fun int8.doSome2() {} +fun int16.doSome2() {} + +fun main() { + int.doSome2(); +} + +/** +@compilation_should_fail +@stderr call to method `doSome2` for type `int` is ambiguous +@stderr candidate function: `int8.doSome2` +@stderr candidate function: `int16.doSome2` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4902.tolk b/tolk-tester/tests/invalid-semantics/err-4902.tolk new file mode 100644 index 000000000..c56be06cc --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4902.tolk @@ -0,0 +1,18 @@ +struct MyMap + +fun MyMap.amb(self) { return 0 } +fun MyMap.amb(self) { return 0 } + +fun main() { + MyMap{}.amb(); // ok + MyMap{}.amb(); // ok + + MyMap{}.amb(); +} + +/** +@compilation_should_fail +@stderr call to method `amb` for type `MyMap` is ambiguous +@stderr candidate function: `MyMap.amb` with K=`int` +@stderr candidate function: `MyMap.amb` with V=`slice` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4903.tolk b/tolk-tester/tests/invalid-semantics/err-4903.tolk new file mode 100644 index 000000000..4c8ab33f5 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4903.tolk @@ -0,0 +1,17 @@ +struct MyMap + +fun MyMap.amb(self) { return 0 } +fun MyMap.amb(self) { return 0 } + +fun main() { + MyMap<(int, int, MyMap), slice>{}.amb(); // ok + MyMap{}.amb(); // ok + + MyMap{}.amb(); +} + +/** +@compilation_should_fail +@stderr method `amb` not found for type `MyMap` +@stderr (but it exists for type `MyMap`) + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4904.tolk b/tolk-tester/tests/invalid-semantics/err-4904.tolk new file mode 100644 index 000000000..062fa5af2 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4904.tolk @@ -0,0 +1,19 @@ +type AssetBuilder = builder +type BalanceBuilder = builder + +fun AssetBuilder.validate(self) { return 0 } +fun BalanceBuilder.validate(self) { return 0 } + +fun main() { + (beginCell() as AssetBuilder).validate(); // ok + (beginCell() as BalanceBuilder).validate(); // ok + + beginCell().validate(); +} + +/** +@compilation_should_fail +@stderr call to method `validate` for type `builder` is ambiguous +@stderr candidate function: `AssetBuilder.validate` +@stderr candidate function: `BalanceBuilder.validate` + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2190.tolk b/tolk-tester/tests/invalid-symbol/err-2190.tolk new file mode 100644 index 000000000..b15475538 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2190.tolk @@ -0,0 +1,14 @@ +enum Color { + Red, + Green, + Blue, +} + +fun main(c: Color) { + c.Red +} + +/** +@compilation_should_fail +@stderr field `Red` doesn't exist in type `Color` + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2258.tolk b/tolk-tester/tests/invalid-symbol/err-2258.tolk new file mode 100644 index 000000000..98e422472 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2258.tolk @@ -0,0 +1,12 @@ +struct Wrapper { item: T; } +fun Wrapper.createNull(): Wrapper { return { item: null } } + +fun main() { + Wrapper.createNull(); +} + +/** +@compilation_should_fail +@stderr method `createNull` not found for type `Wrapper` +@stderr (but it exists for type `Wrapper`) + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2618.tolk b/tolk-tester/tests/invalid-symbol/err-2618.tolk new file mode 100644 index 000000000..29745c2de --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2618.tolk @@ -0,0 +1,9 @@ +enum Unknown { + A = KKK, +} + +/** +@compilation_should_fail +@stderr undefined symbol `KKK` +@stderr A = + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2910.tolk b/tolk-tester/tests/invalid-symbol/err-2910.tolk new file mode 100644 index 000000000..1b5bc4d3f --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2910.tolk @@ -0,0 +1,10 @@ +enum Color { Red, Green, Blue } + +fun main() { + Color.Yellow +} + +/** +@compilation_should_fail +@stderr member `Yellow` does not exist in enum `Color` + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2980.tolk b/tolk-tester/tests/invalid-symbol/err-2980.tolk index de29cc657..65e87ff4a 100644 --- a/tolk-tester/tests/invalid-symbol/err-2980.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2980.tolk @@ -6,6 +6,6 @@ @compilation_should_fail On Linux/Mac, `realpath()` returns an error, and the error message is "cannot find file" On Windows, it fails after, on reading, with a message "cannot open file" -@stderr err-2980.tolk:2:7: error: Failed to import: cannot +@stderr err-2980.tolk:2:7: error: Failed to import: @stderr import "unexisting.tolk"; */ diff --git a/tolk-tester/tests/invalid-syntax/err-3410.tolk b/tolk-tester/tests/invalid-syntax/err-3410.tolk new file mode 100644 index 000000000..354a82de2 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3410.tolk @@ -0,0 +1,11 @@ +enum Color { + Red, + Green, + Blue, + , +} + +/** +@compilation_should_fail +@stderr expected member name, got `,` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3710.tolk b/tolk-tester/tests/invalid-syntax/err-3710.tolk new file mode 100644 index 000000000..00ff2e865 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3710.tolk @@ -0,0 +1,11 @@ +struct A { + private readonly a: () + readonly b: () + private private e: () +} + +/** +@compilation_should_fail +@stderr expected field name, got `private` +@stderr private private e + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3781.tolk b/tolk-tester/tests/invalid-syntax/err-3781.tolk deleted file mode 100644 index bf23d8570..000000000 --- a/tolk-tester/tests/invalid-syntax/err-3781.tolk +++ /dev/null @@ -1,6 +0,0 @@ -enum MyKind { } - -/** -@compilation_should_fail -@stderr `enum` is not supported yet -*/ diff --git a/tolk-tester/tests/invalid-typing/err-6120.tolk b/tolk-tester/tests/invalid-typing/err-6120.tolk new file mode 100644 index 000000000..ebe021ea6 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6120.tolk @@ -0,0 +1,16 @@ +enum Color { Red, Green, Blue } + +fun main(c: Color) { + match (c) { + Color.Red => {} + else => {} + } + match (c) { + Color.Green => {} + } +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible enum members; missing members are: Red, Blue + */ diff --git a/tolk-tester/tests/invalid-typing/err-6121.tolk b/tolk-tester/tests/invalid-typing/err-6121.tolk new file mode 100644 index 000000000..25dd91b63 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6121.tolk @@ -0,0 +1,17 @@ +enum Color { Red, Green, Blue } +type ColorAlias = Color + +fun main(c: Color) { + match (c) { + Color.Red => {} + Color.Green => {} + ColorAlias.Red => {} + else => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: duplicated enum member in `match` +@stderr ColorAlias.Red + */ diff --git a/tolk-tester/tests/invalid-typing/err-6122.tolk b/tolk-tester/tests/invalid-typing/err-6122.tolk new file mode 100644 index 000000000..66e218b64 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6122.tolk @@ -0,0 +1,17 @@ +enum Color { Red, Green, Blue } +type ColorAlias = Color + +fun main(c: Color) { + match (c) { + Color.Red => {} + Color.Green => {} + ColorAlias.Blue => {} + else => {} + } +} + +/** +@compilation_should_fail +@stderr `match` already covers all possible enum members, `else` is invalid +@stderr else => {} + */ diff --git a/tolk-tester/tests/invalid-typing/err-6274.tolk b/tolk-tester/tests/invalid-typing/err-6274.tolk index d586efe41..d436c67c6 100644 --- a/tolk-tester/tests/invalid-typing/err-6274.tolk +++ b/tolk-tester/tests/invalid-typing/err-6274.tolk @@ -14,6 +14,6 @@ fun cantMatchAgainstNotVariant(a: int | slice | MBuilder) { /** @compilation_should_fail -@stderr wrong pattern matching: `cell` is not a variant of `int | slice | MBuilder` +@stderr wrong pattern matching: `MCell` is not a variant of `int | slice | MBuilder` @stderr MCell */ diff --git a/tolk-tester/tests/invalid-typing/err-6310.tolk b/tolk-tester/tests/invalid-typing/err-6310.tolk new file mode 100644 index 000000000..901a04b6b --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6310.tolk @@ -0,0 +1,12 @@ +enum Color { Red, Green, Blue } + +fun main(c: Color) { + match (c) { + 2 => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: can not compare type `int` with match subject of type `Color` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6311.tolk b/tolk-tester/tests/invalid-typing/err-6311.tolk new file mode 100644 index 000000000..56adc34cf --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6311.tolk @@ -0,0 +1,12 @@ +enum Color { Red, Green, Blue } + +fun main() { + match (2 as int8) { + Color.Red => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: can not compare type `Color` with match subject of type `int8` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6312.tolk b/tolk-tester/tests/invalid-typing/err-6312.tolk new file mode 100644 index 000000000..7b9c0dfb1 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6312.tolk @@ -0,0 +1,14 @@ +enum Color { Red, Green, Blue } +enum Shape { Circle, Square } + +fun main(c: Color) { + c as Shape; // ok + match (c) { + Shape.Circle => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: can not compare type `Shape` with match subject of type `Color` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6427.tolk b/tolk-tester/tests/invalid-typing/err-6427.tolk index 37df89a1e..57d295110 100644 --- a/tolk-tester/tests/invalid-typing/err-6427.tolk +++ b/tolk-tester/tests/invalid-typing/err-6427.tolk @@ -1,4 +1,5 @@ fun cantReturnVoidAndNonVoid(a: int | slice | builder) { + throw 123; match (a) { int => { return; } slice => {} diff --git a/tolk-tester/tests/invalid-typing/err-6511.tolk b/tolk-tester/tests/invalid-typing/err-6511.tolk new file mode 100644 index 000000000..880587674 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6511.tolk @@ -0,0 +1,10 @@ +enum Color { Red, Green, Blue } + +fun main(c: Color) { + c == 1 +} + +/** +@compilation_should_fail +@stderr can not apply operator `==` to `Color` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6512.tolk b/tolk-tester/tests/invalid-typing/err-6512.tolk new file mode 100644 index 000000000..03b81b5f6 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6512.tolk @@ -0,0 +1,10 @@ +enum Color { Red, Green, Blue } + +fun main(c: Color) { + c + 1 +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `Color` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6513.tolk b/tolk-tester/tests/invalid-typing/err-6513.tolk new file mode 100644 index 000000000..e68bf89e5 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6513.tolk @@ -0,0 +1,23 @@ +type UserId = int +type OwnerId = int + +fun main() { + var a = 10 as UserId | slice; + match (a) { + UserId => {} + slice => {} + } + __expect_type(a, "UserId | slice"); + + match (a) { + OwnerId => {} + slice => {} + } + return a; +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: `OwnerId` is not a variant of `UserId | slice` +@stderr OwnerId => {} + */ diff --git a/tolk-tester/tests/invalid-typing/err-6593.tolk b/tolk-tester/tests/invalid-typing/err-6593.tolk new file mode 100644 index 000000000..f6e11190e --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6593.tolk @@ -0,0 +1,13 @@ +type UserId = int +type OwnerId = int + +fun f(v: UserId) { + match (v) { + OwnerId => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: `OwnerId` is not a variant of `UserId` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6639.tolk b/tolk-tester/tests/invalid-typing/err-6639.tolk new file mode 100644 index 000000000..e642ad642 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6639.tolk @@ -0,0 +1,12 @@ + +fun main() { + createEmptyMap(); +} + +/** +@compilation_should_fail +@stderr invalid `map`: type `slice` can not be used as a key +@stderr because it does not specify keyLen for a dictionary +@stderr hint: use `address` if a key is an internal address +@stderr hint: use `bits128` and similar if a key represents fixed-width data + */ diff --git a/tolk-tester/tests/invalid-typing/err-6728.tolk b/tolk-tester/tests/invalid-typing/err-6728.tolk new file mode 100644 index 000000000..6da940f0a --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6728.tolk @@ -0,0 +1,17 @@ +struct Point { + x: int8 + y: int8 +} + +fun Cell.myMethod(self, v: U): int + asm "DROP" "HASHCU" + +fun main() { + Point{x:10,y:20}.toCell().myMethod(4); // ok + Point{x:10,y:20}.toCell().myMethod((4, 5)); +} + +/** +@compilation_should_fail +@stderr can not call `Cell.myMethod` with U=(int, int), because it occupies 2 stack slots in TVM, not 1 + */ diff --git a/tolk-tester/tests/invalid-typing/err-6761.tolk b/tolk-tester/tests/invalid-typing/err-6761.tolk new file mode 100644 index 000000000..48bfb12f3 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6761.tolk @@ -0,0 +1,15 @@ +struct Point { + x: int8 + y: int32 +} + + +fun main() { + var m = createEmptyMap(); + m.set(123, createAddressNone()); +} + +/** +@compilation_should_fail +@stderr can not pass `int` to `Point` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6802.tolk b/tolk-tester/tests/invalid-typing/err-6802.tolk new file mode 100644 index 000000000..4511252dc --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6802.tolk @@ -0,0 +1,10 @@ + +fun main() { + var m = createEmptyMap(); + m.set(5, ""); +} + +/** +@compilation_should_fail +@stderr can not pass `slice` to `int32` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6834.tolk b/tolk-tester/tests/invalid-typing/err-6834.tolk new file mode 100644 index 000000000..9936e96fa --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6834.tolk @@ -0,0 +1,14 @@ +type SnakedCell = cell + +type MInt_v1 = int +type MInt_v2 = int + +fun m(a: SnakedCell, b: SnakedCell) { + a = b +} + +/** +@compilation_should_fail +@stderr can not assign `SnakedCell` to variable of type `SnakedCell` +@stderr hint: use `as` operator for unsafe casting: ` as SnakedCell` + */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index f686fe67c..51ec9148d 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1591,6 +1591,14 @@ fun demo411(s: slice) { } } +@method_id(412) +fun demo412() { + val msg = lazy AOrB.fromSlice(""); + match (msg) { + } + return 123; +} + fun main() { val s = stringHexToSlice("0102"); @@ -1628,10 +1636,10 @@ fun main() { @testcase | 113 | x{03ffff} | -1 @testcase | 113 | x{04000000FF} | 255 @testcase | 114 | x{0110} | 16 1 -@testcase | 114 | x{020f} | [ -15 ] typeid-27 +@testcase | 114 | x{020f} | [ -15 ] typeid-2 @testcase | 114 | x{03} | -1 1 @testcase | 114 | x{03ffff} | -1 1 -@testcase | 114 | x{04000000FF} | [ 255 ] typeid-27 +@testcase | 114 | x{04000000FF} | [ 255 ] typeid-2 @testcase | 115 | x{08} | 8 42 @testcase | 115 | x{88} | 0 2 @testcase | 115 | x{83} | -1 2 @@ -1646,9 +1654,9 @@ fun main() { @testcase | 122 | x{010288} | -8 1 @testcase | 123 | x{010280} | 5 @testcase | 124 | x{010283} | [ -3 2 ] -@testcase | 125 | | 127 32 15 2 3 typeid-1 15 3 -1 -@testcase | 126 | x{7F0F20} | 127 (null) (null) 15 typeid-3 32 15 -1 -@testcase | 126 | x{7F8F020320} | 127 15 2 3 typeid-4 32 -1 2 +@testcase | 125 | | 127 32 15 2 3 typeid-6 15 3 -1 +@testcase | 126 | x{7F0F20} | 127 (null) (null) 15 typeid-8 32 15 -1 +@testcase | 126 | x{7F8F020320} | 127 15 2 3 typeid-7 32 -1 2 @testcase | 127 | | 1 2 -1 255 @testcase | 128 | x{5500} | -1 -1 @testcase | 128 | x{558F01024} | 0 2 @@ -1755,6 +1763,7 @@ fun main() { @testcase | 407 | x{BF} | -1 @testcase | 408 | | 24 @testcase | 409 | | 255 3 +@testcase | 412 | | 123 @fif_codegen """ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 9b70ae40c..0fcd3f25c 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -1,9 +1,11 @@ import "imports/use-dicts.ext.tolk" +@method_id(90) fun simpleAllConst() { return (!0, !!0 & !false, !!!0, !1, !!1, !-1, !!-1, (!5 as int == 0) == !0, !0 == true); } +@method_id(91) fun compileTimeEval1(x: int) { // todo now compiler doesn't understand that bool can't be equal to number other than 0/-1 // (but understands that it can't be positive) diff --git a/tolk-tester/tests/maps-tests.tolk b/tolk-tester/tests/maps-tests.tolk new file mode 100644 index 000000000..e80d9c01b --- /dev/null +++ b/tolk-tester/tests/maps-tests.tolk @@ -0,0 +1,704 @@ + +struct JustInt32 { + v: MInt32 +} + +struct JustUint64 { + v: uint64 +} + +struct JustAddress { + v: address +} + +type MInt32 = int32 +type MSlice = slice + +struct Point { + x: int8 + y: int8 +} + +struct Point3d { + x: int8 + y: int8 + z: int8 +} + +struct DataWithCell { + a: address + v: coins + p: Cell? + payload: cell? = null +} + +enum HashMode: uint16 { + NoHash = 0, + PartialHash = 80, + TotalHash = 255, +} + +fun map.calculateSize(self) { + var size = 0; + var r = self.findFirst(); + while (r.isFound) { + size += 1; + r = self.iterateNext(r); + } + return size; +} + + +@method_id(101) +fun test1() { + var m: map = createEmptyMap(); + m.set(1, 10); + return m.exists(1); +} + +@method_id(102) +fun test2(k1: int, v1: int, k2: int, v2: int) { + var m: map = createMapFromLowLevelDict(null); + m.set(k1, v1); + m.set(k2, v2); + return (m.exists(1), m.exists(k1), m.mustGet(k2)); +} + +@method_id(103) +fun test3(v1: int) { + var m = createEmptyMap(); + m.set(1, (1, 2)); + m.set(2, (3, v1)); + m.delete(1); + return (m.exists(1), m.mustGet(2)); +} + +@method_id(104) +fun test4() { + var m = createEmptyMap(); + m.set(1, 10); + m.set(2, 20); + m.set(3, 30); + val g1 = m.get(1); + val g4 = m.get(4); + return (g1.isFound, g4.isFound, g1.loadValue()); +} + +@method_id(105) +fun test5() { + var m = createEmptyMap(); + m.set(1, 10); + m.set(2, 20); + m.set(3, 30); + var t = createEmptyTuple(); + var r = m.findFirst(); + while (r.isFound) { + t.push(r.getKey()); + t.push(r.loadValue()); + r = m.iterateNext(r); + } + r = m.findLast(); + while (r.isFound) { + t.push(r.loadValue()); + r = m.iterateNext(r); + } + r = m.findLast(); + while (r.isFound) { + t.push(r.loadValue()); + r = m.iteratePrev(r); + } + return t; +} + +@method_id(106) +fun test6() { + var m = createEmptyMap(); + m.set(1, 10); + m.set(2, 20); + m.set(3, 30); + return ( + m.findKeyGreater(1).getKey(), m.findKeyGreaterOrEqual(1).getKey(), + m.findKeyLess(3).getKey(), m.findKeyLessOrEqual(3).getKey(), + m.findKeyLess(3).loadValue(), m.findKeyLessOrEqual(3).loadValue(), + m.findKeyLessOrEqual(3).isFound, m.findKeyGreater(4).isFound, + ) +} + +@method_id(107) +fun test7() { + var m1 = createEmptyMap(); + m1.addIfNotExists(1, true); + m1.addIfNotExists(1, false); + m1.replaceIfExists(2, true); + return (m1.mustGet(1), m1.replaceIfExists(2, false), m1.get(2).isFound); +} + +@method_id(108) +fun test8() { + var (a, m: map) = (1, createEmptyMap()); + val prev1 = m.setAndGetPrevious(a, createAddressNone()); + val prev2 = m.setAndGetPrevious(a, address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5")); + return (prev1.isFound, prev2.isFound, prev2.loadValue().isNone(), prev2.loadValue().isNone(), m.get(a).loadValue().getWorkchain()); +} + +@method_id(109) +fun test9() { + var m = createEmptyMap(); + m.set(1, (ton("1"), ())); + val replaced = m.replaceIfExists(1, (ton("0.05"), ())); + val prev = m.replaceAndGetPrevious(1, (ton("0.01"), ())); + return (replaced, prev.loadValue(), m.replaceAndGetPrevious(2, (0,())), m.mustGet(1).0); +} + +@method_id(110) +fun test10() { + var m = createEmptyMap(); + val added1 = m.addIfNotExists(1<<100, 1<<10); + val added2 = m.addIfNotExists(1<<100, 2<<10); + val b1 = m.addOrGetExisting(2<<100, 20); + val b2 = m.addOrGetExisting(2<<100, 30); + return (added1, added2, m.mustGet(1<<100), b1.isFound, b2.isFound, b2.loadValue()); +} + +@method_id(111) +fun test11() { + var m: map; + m = createEmptyMap(); + m.set(0, null); + m.set(1, 1); + m.set(2, null); + return (m.exists(0), m.exists(1), m.mustGet(0), m.mustGet(1), m.get(0).isFound, m.get(0).loadValue(), m.get(3)); +} + +@method_id(112) +fun test12() { + var m1: map = createEmptyMap(); + val s1 = m1.calculateSize(); + m1.set(1, 10); + m1.addIfNotExists(1, -100); + val s2 = m1.calculateSize(); + var m2 = m1; + m2.addIfNotExists(2, 20); + m2.addIfNotExists(3, 30); + var m3 = m2; + val wasDeleted = m2.delete(2); + var del1 = m1.deleteAndGetDeleted(-100); + var del2 = m2.deleteAndGetDeleted(2); + var del3 = m2.deleteAndGetDeleted(3); + return ( + s1, s2, m3.calculateSize(), + wasDeleted, del1.isFound, del2.isFound, del3.isFound, + del3.loadValue(), + ); +} + +@method_id(113) +fun test13(m: map) { + // key remains just int at the TVM level, no runtime packing, checked via codegen + return m.get({v: 123}) +} + +@method_id(114) +fun test14(fromV: int) { + var m = createEmptyMap(); + m.set({v: 1}, 10); + m.set({v: 2}, 20); + m.set({v: 3}, 30); + return m.findKeyGreater({v: fromV}).getKey() +} + +@method_id(115) +fun test15() { + var m = createEmptyMap(); + m.set({v: address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA")}, 10); + m.set({v: address("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6")}, 20); + m.set({v: address("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y")}, 30); + return m.mustGet({v:address("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6")}); +} + +@method_id(116) +fun test16() { + var m = createEmptyMap(); + m.set(100, {x: 10, y: 20}); + return (m.deleteAndGetDeleted(100).loadValue(), m.calculateSize(), m.isEmpty()); +} + +@method_id(117) +fun test17() { + var m = createEmptyMap(); + m.set(true, 123); + return m.findFirst().getKey(); +} + +@method_id(118) +fun test18() { + var m: map = createEmptyMap(); + m.set({x: 1, y: 1}, {x: 10, y: 10}); + m.addIfNotExists({x: 1, y: 1}, {x: -100, y: -100}); + val f1 = m.get({x: 1, y: 1}); + m.replaceIfExists({x: 1, y: 1}, {x: 20, y: 20}); + val f2 = m.get({x: 1, y: 1}); + val deleted1 = m.delete({x: 1, y: 1}); + val deleted2 = m.delete({x: 1, y: 1}); + val f3 = m.get({x: 1, y: 1}); + return (m, f1.isFound, f2.isFound, f3.isFound, f1.loadValue(), f2.loadValue(), deleted1, deleted2, m.isEmpty()); +} + +@method_id(119) +fun test19() { + var m: map> = createEmptyMap(); + m.addIfNotExists(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), Point{x:10,y:10}.toCell()); + return m.get(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA")).loadValue().load(); +} + +@method_id(120) +fun test20() { + var m: map> = createEmptyMap(); + m.addIfNotExists(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), Point{x:10,y:10}.toCell()); + return m.mustGet(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA")).load(); +} + +@method_id(121) +fun test21() { + var m: map> = createEmptyMap(); + val r1 = m.addOrGetExisting(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), Point{x:10,y:10}.toCell()); + val r2 = m.addOrGetExisting(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), Point{x:20,y:20}.toCell()); + val d1 = m.deleteAndGetDeleted(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA")); + val d2 = m.deleteAndGetDeleted(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA")); + return (r1.isFound, r2.isFound, r2.loadValue().load(), d1.isFound, d2.isFound); +} + +@method_id(122) +fun test22() { + var m: map> = createEmptyMap(); + m.set(3, Point{x:30,y:30}.toCell()); + m.set(2, Point{x:20,y:20}.toCell()); + m.set(1, Point{x:10,y:10}.toCell()); + return (m.findFirst().getKey(), m.findLast().getKey(), m.findKeyGreater(2).getKey(), m.findKeyGreaterOrEqual(2).loadValue().load(), m.mustGet(2).load().x); +} + +@method_id(123) +fun test23() { + var m: map = createEmptyMap(); + val p1 = m.setAndGetPrevious({x:1,y:1}, {a: createAddressNone(), v: ton("0.05"), p: Point{x:10,y:10}.toCell(), payload: null}); + val p2 = m.setAndGetPrevious({x:1,y:1}, {a: createAddressNone(), v: ton("0.1"), p: null, payload: createEmptyCell()}); + return (p1.isFound, p2.isFound, p2.loadValue().p!.load(), m.mustGet({x:1,y:1}).a.isNone(), m.get({x:1,y:1}).loadValue().payload.depth()); +} + +@method_id(124) +fun test24() { + var m = createEmptyMap(); + val p1 = m.deleteAndGetDeleted({x: 1, y: 1}); + m.set({x: 3, y: 3}, {a: createAddressNone(), v: ton("0.05"), p: Point{x:30,y:30}.toCell(), payload: createEmptyCell()}); + m.set({x: 1, y: 1}, {a: createAddressNone(), v: ton("0.05"), p: Point{x:10,y:10}.toCell(), payload: null}); + m.set({x: 2, y: 2}, {a: createAddressNone(), v: ton("0.05"), p: null, payload: null}); + val k2 = m.findKeyGreaterOrEqual({x: 2, y: 2}).getKey(); + var r = m.findFirst(); + var t = createEmptyTuple(); + while (r.isFound) { + t.push(r.getKey().x); + t.push(r.getKey().y); + t.push(r.loadValue().payload == null); + r = m.iterateNext(r); + } + return (p1, k2, t); +} + +@method_id(125) +fun test25() { + var m = createEmptyMap(); + m.set(1, stringHexToSlice("0101")); + return m; +} + +@method_id(126) +fun test26() { + var m = createEmptyMap(); + m.set(1, stringHexToSlice("0101")); + val f1 = m.setAndGetPrevious(1, stringHexToSlice("010101")); + val f2 = m.deleteAndGetDeleted(1); + val f3 = m.setAndGetPrevious(1, stringHexToSlice("01010101")); + return ( + f1.isFound, f2.isFound, f3.isFound, + f1.loadValue().remainingBitsCount(), f2.loadValue().remainingBitsCount(), + m.mustGet(1).remainingBitsCount(), + ); +} + +@method_id(127) +fun test27() { + var m = createEmptyMap(); + m.set(1, ""); + return (m.findFirst().getKey(), m.findFirst().loadValue().remainingBitsCount()); +} + +@method_id(128) +fun test28() { + var m = createEmptyMap(); + m.set({x: 1, y: 1}, stringHexToSlice("01")); + m.set({x: 3, y: 3}, stringHexToSlice("03")); + m.set({x: 2, y: 2}, stringHexToSlice("02")); + var r = m.findKeyLessOrEqual({x: 4, y: 0}); + var sum = 0; + while (r.isFound) { + sum += r.loadValue().loadUint(8); + r = m.iteratePrev(r); + } + return (sum, m.calculateSize()); +} + +@method_id(129) +fun test29() { + var m = createEmptyMap(); + val a1 = m.addIfNotExists(1, DataWithCell{a: createAddressNone(), v: 0, p: null, payload: null}.toCell().beginParse()); + val a2 = m.addIfNotExists(2, DataWithCell{a: createAddressNone(), v: 0, p: Point{x:20,y:20}.toCell(), payload: null}.toCell().beginParse()); + val a3 = m.addIfNotExists(3, DataWithCell{a: createAddressNone(), v: 0, p: Point{x:30,y:30}.toCell(), payload: createEmptyCell()}.toCell().beginParse()); + val r4 = m.replaceAndGetPrevious(4, ""); + return ( + a1, a2, a3, r4, + DataWithCell.fromSlice(m.mustGet(2)).payload, + m.get(3).loadValue().loadAny().p!.load(), + m.get(3).loadValue().remainingRefsCount(), + m.findLast().getKey(), + m.findLast().loadValue().remainingRefsCount(), + ); +} + +@method_id(130) +fun test30() { + var m = createEmptyMap(); + m.set(1, createEmptyCell()); + m.set(2, Point{x:20,y:20}.toCell()); + m.set(3, Point{x:30,y:30}.toCell()); + val c3 = m.get(3); + var r = m.findKeyLess(3); + var t = createEmptyTuple(); + while (r.isFound) { + t.push(r.loadValue().beginParse().remainingBitsCount()); + r = m.iteratePrev(r); + } + val p2 = lazy Point.fromCell(m.mustGet(2)); + return (c3.isFound, c3.loadValue().beginParse().loadAny(), t, p2.y); +} + +@method_id(131) +fun test31() { + var m = createEmptyMap(); + m.set(1, createEmptyCell()); + m.set(2, Point{x:20,y:20}.toCell()); + m.set(3, Point{x:30,y:30}.toCell()); + m.set(4, null); + val c4 = m.get(4); + val c5 = m.get(5); + var r = m.findLast(); + var t = createEmptyTuple(); + while (r.isFound) { + t.push(r.loadValue() == null); + r = m.iteratePrev(r); + } + return (m.mustGet(4), c4.loadValue(), c5.isFound, t); +} + +@method_id(132) +fun test32() { + var m = createEmptyMap(); + m.set(stringHexToSlice("0102") as bits16, stringHexToSlice("aaaa") as bits16); + m.set(stringHexToSlice("0304") as bits16, stringHexToSlice("bbbb") as bits16); + val repl = m.replaceAndGetPrevious(stringHexToSlice("0304") as bits16, stringHexToSlice("0000") as bits16); + var r = m.iteratePrev(m.findLast()); + return ((r.getKey() as slice).remainingBitsCount(), (r.getKey() as slice).loadAny(), (r.loadValue() as slice).remainingBitsCount(), repl.isFound, (repl.loadValue() as slice).loadUint(8)); +} + +@method_id(133) +fun test33() { + var m = createEmptyMap(); + m.set(123, {x: 10, y: 10}); + return m.mustGet(123); +} + +@method_id(134) +fun test34(z: int) { + var m = createEmptyMap(); + m.set(123, {x: 10, y: 10, z}); + return [m.mustGet(123).z, m.addIfNotExists(456, {x: 10, y: z, z}), m.mustGet(456).y]; +} + +@method_id(135) +fun test35() { + var m: map> = createEmptyMap(); + m.set(1, createEmptyMap()); + val f1 = m.mustGet(1).findFirst(); + assert (!f1.isFound) throw 123; + m.get(1).loadValue().set(10, 100); + assert (m.mustGet(1).isEmpty()) throw 123; // the previous set() changed a copy + + var nested = createEmptyMap(); + nested.set(1,10).set(2, 20); + m.replaceIfExists(1, nested); + nested.set(3, 30); + assert (nested.calculateSize() == 3) throw 123; + assert (m.mustGet(1).calculateSize() == 2) throw 123; + + val (k1, v2) = (m.mustGet(1).findFirst().getKey(), m.get(1).loadValue().findLast().loadValue()); + val s1 = m.findFirst().loadValue().calculateSize(); + m.mustGet(1).delete(10); + val d2 = m.mustGet(1).deleteAndGetDeleted(2); + return (f1.isFound, k1, v2, s1, d2.loadValue(), m.mustGet(1).isEmpty(), m.get(1).loadValue().calculateSize()); +} + +@method_id(136) +fun test36() { + val k = address("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"); + var m = createEmptyMap(); + m.set(k, 10); + return m.mustGet(k); +} + +@method_id(137) +fun test37() { + try { + var m = createEmptyMap(); + m.set(1, 10); + return m.mustGet(555) as int; + } catch (ex) { + return ex; + } +} + +@method_id(138) +fun test38(k: int) { + try { + var m = createEmptyMap(); + m.set(1, 10); + return m.mustGet(k, 55); + } catch (ex) { + return ex as int64; + } +} + +type Set = map + +fun createEmptySet(): Set { return createEmptyMap() } +fun Set.add(mutate self, k: T) { self.set(k, ()) } +fun Set.remove(mutate self, k: T) { self.delete(k) } + +@method_id(139) +fun test39() { + var s: Set = createEmptySet(); + s.add(10); s.add(10); + s.add(20); + s.add(30); + s.remove(20); + return (s.exists(20), s.exists(10), s.calculateSize()); +} + +@method_id(140) +fun test40() { + var `m!empty` = createEmptyMap() as map?; + var `m!filled` = createEmptyMap() as map?; + var `m_null` = null as map?; + + `m!filled`!.set(10, 20); + return ( + `m!empty` == null, `m!filled` == null, `m_null` == null, 777, + `m!empty`!.isEmpty(), `m!filled`!.isEmpty(), 777, + `m!empty`, `m_null`, + ); +} + +@method_id(141) +fun test41() { + var m = createEmptyMap(); + m.set(-7, HashMode.NoHash); + m.addIfNotExists(111, HashMode.TotalHash); + m.replaceIfExists(-7, HashMode.PartialHash); + + val k7 = m.get(-7); + return ( + m.exists(8), m.mustGet(111) == HashMode.TotalHash, + k7.isFound, k7.loadValue() == HashMode.NoHash, k7.loadValue() == HashMode.PartialHash + ); +} + + +fun main() { + __expect_type(createEmptyMap, "() -> map"); +} + +/** +@testcase | 101 | | -1 +@testcase | 102 | 1 10 2 20 | -1 -1 20 +@testcase | 102 | 2 20 3 30 | 0 -1 30 +@testcase | 103 | 100 | 0 3 100 +@testcase | 104 | | -1 0 10 +@testcase | 105 | | [ 1 10 2 20 3 30 30 30 20 10 ] +@testcase | 106 | | 2 1 2 3 20 30 -1 0 +@testcase | 107 | | -1 0 0 +@testcase | 108 | | 0 -1 -1 -1 0 +@testcase | 109 | | -1 50000000 (null) 0 10000000 +@testcase | 110 | | -1 0 1024 0 -1 20 +@testcase | 111 | | -1 -1 (null) 1 -1 (null) (null) 0 +@testcase | 112 | | 0 1 3 -1 0 0 -1 30 +@testcase | 114 | 2 | 3 +@testcase | 114 | 3 | (null) +@testcase | 115 | | 20 +@testcase | 116 | | 10 20 0 -1 +@testcase | 117 | | -1 +@testcase | 118 | | (null) -1 -1 0 10 10 20 20 -1 0 -1 +@testcase | 119 | | 10 10 +@testcase | 120 | | 10 10 +@testcase | 121 | | 0 -1 10 10 -1 0 +@testcase | 122 | | 1 3 3 20 20 20 +@testcase | 123 | | 0 -1 10 10 -1 0 +@testcase | 124 | | (null) 0 2 2 [ 1 1 -1 2 2 -1 3 3 0 ] +@testcase | 126 | | -1 -1 0 16 24 32 +@testcase | 127 | | 1 0 +@testcase | 128 | | 6 3 +@testcase | 129 | | -1 -1 -1 (null) 0 (null) 30 30 2 3 2 +@testcase | 130 | | -1 30 30 [ 16 0 ] 20 +@testcase | 131 | | (null) (null) 0 [ -1 0 0 0 ] +@testcase | 132 | | 16 1 2 16 -1 187 +@testcase | 133 | | 10 10 +@testcase | 134 | 5 | [ 5 -1 5 ] +@testcase | 135 | | 0 1 20 2 20 0 2 +@testcase | 136 | | 10 +@testcase | 137 | | 9 +@testcase | 138 | 1 | 10 +@testcase | 138 | 4 | 55 +@testcase | 139 | | 0 -1 2 +@testcase | 140 | | 0 0 -1 777 -1 0 777 (null) typeid-1 (null) 0 +@testcase | 141 | | 0 -1 -1 0 -1 + +@fif_codegen +""" + test1() PROC:<{ // + NEWDICT // m + 1 PUSHINT // m '2=1 + x{0000000a} PUSHSLICE + s0 s2 XCHG + 8 PUSHINT + DICTISET // m + 1 PUSHINT // m '11=1 + SWAP + 8 PUSHINT // '11=1 m '16=8 + DICTIGET NULLSWAPIFNOT // '12 '13 + NIP // '13 + }> +""" + +@fif_codegen +""" + test13() PROC:<{ // m + 123 PUSHINT // m '1=123 + SWAP + 32 PUSHINT // '1=123 m '6=32 + DICTIGET NULLSWAPIFNOT // '2 '3 + }> +""" + +@fif_codegen +""" + test14() PROC:<{ + ... + 64 PUSHINT + DICTUGETNEXT NULLSWAPIFNOT2 + DROP + NIP + }> +""" + +@fif_codegen +""" + test15() PROC:<{ + ... + x{9FE9794E38E5400E2C75F4BF2DFBAB1BF0568C9EDFC717359BB8999C47D4FB9AC23_} PUSHSLICE + SWAP + 267 PUSHINT + DICTGET + 9 THROWIFNOT + 8 LDI + ENDS + }> +""" + +@fif_codegen +""" + test16() PROC:<{ + NEWDICT + 100 PUSHINT + x{0a14} PUSHSLICE + s0 s2 XCHG + 32 PUSHINT + DICTISET +""" + +@fif_codegen +""" + test25() PROC:<{ // + NEWDICT // m + 1 PUSHINT // m '2=1 + x{0101} PUSHSLICE // m '2=1 '3 + s0 s2 XCHG + 8 PUSHINT // '3 '2=1 m '6=8 + DICTISET // m + }> +""" + +@fif_codegen +""" + test30() PROC:<{ + NEWDICT + 1 PUSHINT + PUSHREF + s0 s2 XCHG + 8 PUSHINT + DICTISETREF +""" + +@fif_codegen +""" + test33() PROC:<{ + NEWDICT + 123 PUSHINT + x{0a0a} PUSHSLICE + s0 s2 XCHG + 8 PUSHINT + DICTUSET + 123 PUSHINT + SWAP + 8 PUSHINT + DICTUGET + 9 THROWIFNOT + 8 LDI + 8 LDI + ENDS + }> +""" + +@fif_codegen +""" + test34() PROC:<{ + NEWDICT + 123 PUSHINT + NEWC + x{0a0a} STSLICECONST + s3 PUSH + 8 STIR + s0 s2 XCHG + 32 PUSHINT + DICTUSETB +""" + +@fif_codegen +""" + test36() PROC:<{ // + x{9FFDE075922FCD6EC7F0A0F3E32D6828C8AE03395763E4C5ECF19503052F1829F45_} PUSHSLICE // k + NEWDICT // k m + x{0000000a} PUSHSLICE + s2 s1 PUXC + 267 PUSHINT + DICTSET // k m +""" + +*/ + diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 489cdf748..588632051 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -81,9 +81,9 @@ fun test7(x: int8) { @method_id(108) fun test8(x: int) { - var r1 = match (x) { else => 10 }; + var r1 = match (x is int && x is int is bool) { false => 0, else => 10 }; match (x) { } - match (x) { else => { r1 += 20; } } + match (x * 0) { 1 => {} else => { r1 += 20; } } return r1; } diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk index 4b09a53d6..c4d5948bc 100644 --- a/tolk-tester/tests/methods-tests.tolk +++ b/tolk-tester/tests/methods-tests.tolk @@ -191,11 +191,11 @@ fun T?.arbitraryMethod(self): never { throw 123; } fun test12() { __expect_type(10.arbitraryMethod(), "int?"); - __expect_type((10 as int8?).arbitraryMethod(), "int?"); __expect_type((10 as coins?)!.arbitraryMethod(), "int?"); - __expect_type(null.arbitraryMethod(), "int?"); - __expect_type((10 as coins?).arbitraryMethod(), "int?"); + __expect_type((10 as int8?).arbitraryMethod(), "never"); + __expect_type(null.arbitraryMethod(), "never"); + __expect_type((10 as coins?).arbitraryMethod(), "never"); __expect_type(Wrapper { item: 10 }.arbitraryMethod(), "Wrapper"); __expect_type(Wrapper { item: 10 as int8 }.arbitraryMethod(), "Wrapper"); @@ -280,7 +280,7 @@ fun Wrapper.createFrom(item: U): Wrapper { return {item}; } fun test18() { __expect_type(10.copy, "(int) -> int"); __expect_type(Wrapper{item:null as int8?}.copy, "(Wrapper) -> Wrapper"); - __expect_type(Wrapper.createNull, "() -> Wrapper"); + __expect_type(Wrapper.createNull, "() -> Wrapper"); __expect_type(Wrapper?>.createNull, "() -> Wrapper?>"); __expect_type(Wrapper.createFrom, "(int8) -> Wrapper"); @@ -288,6 +288,25 @@ fun test18() { return cb(10); } +type BalanceList = dict +type AssetList = dict + +fun BalanceList.validate(self): int1 { return 0 } +fun AssetList.validate(self): int2 { return 0 } + +fun test19(b: BalanceList, a: AssetList) { + __expect_type(b.validate(), "int1"); + __expect_type(a.validate(), "int2"); + + if (b == null) {} + match (a) { + null => {} + cell => {} + } + __expect_type(b, "BalanceList"); + __expect_type(a, "AssetList"); +} + fun main() {} /** @@ -300,7 +319,7 @@ fun main() {} @testcase | 107 | | 2 1 -1 @testcase | 108 | 5 | 17 5 @testcase | 109 | | 5 100500 20 30 5 5 31 -@testcase | 110 | | [ 2 3 1 1 3 1 1 ] -1 +@testcase | 110 | | [ 2 3 1 1 1 1 1 ] -1 @testcase | 111 | | [ 1 2 3 3 2 ] @testcase | 113 | | 0 -1 -1 -1 0 -1 @testcase | 114 | | -1 -1 0 0 0 diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index f3e93b694..b31e91ac8 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -307,7 +307,12 @@ fun testExplicitReturn() { } -fun main(){} +fun main(){ + // mark used to codegen them + load_next; + testStoreUintPureUnusedResult; + testStoreUintImpureUnusedResult +} /** @testcase | 101 | | 70 60 diff --git a/tolk-tester/tests/never-type-tests.tolk b/tolk-tester/tests/never-type-tests.tolk index 894473893..b19d88caf 100644 --- a/tolk-tester/tests/never-type-tests.tolk +++ b/tolk-tester/tests/never-type-tests.tolk @@ -1,5 +1,14 @@ fun takeInt(a: int) {} +fun alwaysThrows() { + throw 123; +} + +fun alwaysThrowsButReturns() { + throw 123; + return 123; +} + @method_id(101) fun test1(x: int?) { if (x == null && x != null) { @@ -21,6 +30,8 @@ fun test1(x: int?) { fun main() { __expect_type(test1, "(int?) -> int"); + __expect_type(alwaysThrows(), "never"); + __expect_type(alwaysThrowsButReturns(), "int"); } /** diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 5b0394542..4aa804b49 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -503,7 +503,7 @@ fun main(){} @testcase | 102 | | 1 2 typeid-1 (null) (null) 0 @testcase | 103 | 1 2 | 3 3 0 1 2 @testcase | 104 | | 1 2 (null) (null) 0 -@testcase | 105 | | (null) (null) (null) 0 1 2 3 typeid-3 +@testcase | 105 | | (null) (null) (null) 0 1 2 3 typeid-2 @testcase | 106 | | 1 2 @testcase | 107 | | 0 0 -1 0 0 -1 @testcase | 108 | 5 6 | 7 8 10 11 typeid-1 (null) (null) 0 @@ -525,33 +525,33 @@ fun main(){} @testcase | 121 | | [ 1 [ 3 4 ] ] @testcase | 122 | 0 | [ 1 [ 3 4 ] 4 (null) ] @testcase | 122 | -1 | [ 1 (null) 4 (null) ] -@testcase | 123 | | 1 3 4 typeid-4 -@testcase | 124 | 0 | 1 3 4 typeid-4 4 (null) (null) 0 +@testcase | 123 | | 1 3 4 typeid-3 +@testcase | 124 | 0 | 1 3 4 typeid-3 4 (null) (null) 0 @testcase | 124 | -1 | 1 (null) (null) 0 4 (null) (null) 0 @testcase | 125 | | 3 @testcase | 126 | | 1 (null) 2 @testcase | 127 | 1 | 1 (null) (null) 0 2 @testcase | 127 | 2 | 1 2 3 typeid-1 4 @testcase | 127 | 3 | 1 (null) (null) 0 5 -@testcase | 128 | 1 | 1 (null) (null) 0 2 typeid-11 +@testcase | 128 | 1 | 1 (null) (null) 0 2 typeid-6 @testcase | 128 | 2 | (null) (null) (null) (null) (null) 0 -@testcase | 128 | 3 | 1 2 3 typeid-1 4 typeid-11 +@testcase | 128 | 3 | 1 2 3 typeid-1 4 typeid-6 @testcase | 129 | 0 | 5 5 0 -1 1 2 0 -1 @testcase | 129 | -1 | 5 5 0 -1 (null) (null) 0 -1 @testcase | 130 | 0 | 1 2 3 typeid-1 @testcase | 130 | -1 | 1 (null) (null) 0 -@testcase | 131 | | typeid-12 777 0 777 777 777 0 0 typeid-12 typeid-12 777 typeid-12 typeid-12 typeid-13 777 +@testcase | 131 | | typeid-7 777 0 777 777 777 0 0 typeid-7 typeid-7 777 typeid-7 typeid-7 typeid-8 777 @testcase | 132 | | -1 0 -1 0 777 (null) (null) -1 0 0 @testcase | 133 | | 60 @testcase | 134 | | 11 21 typeid-1 -@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 typeid-15 (null) typeid-15 (null) 0 777 10 typeid-16 (null) typeid-16 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 typeid-9 (null) typeid-9 (null) 0 777 10 typeid-10 (null) typeid-10 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 -@testcase | 140 | 8 9 | 8 9 typeid-17 (null) (null) 0 +@testcase | 140 | 8 9 | 8 9 typeid-11 (null) (null) 0 @testcase | 141 | | (null) 10 @testcase | 142 | | 3 3 1 2 @testcase | 143 | -1 0 | 1 2 1 3 777 (null) (null) (null) (null) 0 -@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 typeid-18 +@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 typeid-12 @fif_codegen """ diff --git a/tolk-tester/tests/overloads-tests.tolk b/tolk-tester/tests/overloads-tests.tolk new file mode 100644 index 000000000..70baa48b5 --- /dev/null +++ b/tolk-tester/tests/overloads-tests.tolk @@ -0,0 +1,289 @@ +struct Wrapper { item: T } +struct Nothing {} +struct MyPair {} + +struct Point { x: int8; y: int8 } + +type MInt = int +type MSlice = slice +type PointAlias = Point + +fun T.realValue(self): int1 { return 0 } +fun TT?.realValue(self): int2 { return 0 } +fun int.realValue(self): int3 { return 0 } +fun int8.realValue(self): int4 { return 0 } +fun int?.realValue(self): int5 { return 0 } +fun Nothing.realValue(self): int6 { return 0 } +fun Nothing>.realValue(self): int7 { return 0 } +fun Nothing.realValue(self): int8 { return 0 } + +fun test1() { + __expect_type(beginCell().realValue(), "int1"); + __expect_type((5 as int|slice).realValue(), "int1"); + __expect_type((beginCell() as builder?).realValue(), "int2"); + __expect_type((null as null|Nothing).realValue(), "int2"); + __expect_type((null as Nothing|null).realValue(), "int2"); + __expect_type(5.realValue(), "int3"); + __expect_type((5 as int?).realValue(), "int5"); + __expect_type((5 as int8).realValue(), "int4"); + __expect_type((5 as int8?).realValue(), "int2"); + __expect_type(Nothing{}.realValue(), "int6"); + __expect_type(Nothing{}.realValue(), "int6"); + __expect_type(Nothing{}.realValue(), "int8"); + __expect_type(Nothing{}.realValue(), "int8"); + __expect_type(Nothing{}.realValue(), "int6"); + __expect_type(Nothing>{}.realValue(), "int7"); + __expect_type(Nothing>>{}.realValue(), "int7"); + __expect_type(Nothing>?>{}.realValue(), "int8"); +} + +fun T.doSome2(self): int { return 0 } +fun int8.doSome2(self): cell { return createEmptyCell() } +fun int16.doSome2(self): slice { return "" } + +fun test2() { + __expect_type(5.doSome2(), "int"); + __expect_type((5 as int8).doSome2(), "cell"); + __expect_type(((5 as int8) as int16).doSome2(), "slice"); + __expect_type((5 as int?).doSome2(), "int"); + __expect_type(5.doSome2().doSome2(), "int"); + __expect_type((1, 2).doSome2(), "int"); +} + +fun MyPair.mapFn(self): int1 { return 0 } +fun MyPair.mapFn(self): int2 { return 0 } +fun MyPair.mapFn(self): int3 { return 0 } +fun MyPair.mapFn(self): int4 { return 0 } + +fun test3() { + __expect_type(MyPair{}.mapFn(), "int1"); + __expect_type(MyPair{}.mapFn(), "int1"); + __expect_type(MyPair{}.mapFn(), "int2"); + __expect_type(MyPair{}.mapFn(), "int2"); + __expect_type(MyPair{}.mapFn(), "int3"); + __expect_type(MyPair{}.mapFn(), "int4"); + __expect_type(MyPair{}.mapFn(), "int4"); +} + +fun (int|slice).unionMix(self): slice { return "" } +fun int.unionMix(self): bool { return true } + +fun test4() { + __expect_type((5 as int).unionMix(), "bool"); + __expect_type(("".unionMix()), "slice"); + __expect_type(((5 as int|slice).unionMix()), "slice"); +} + +fun WrapA.dup(self): WrapA { return self } +fun WrapA>.dup(self): WrapA> { return self } +struct WrapA {} + +fun test5() { + __expect_type(WrapA{}.dup(), "WrapA"); + __expect_type(WrapA>{}.dup(), "WrapA>"); +} + +type AliasBuilder1 = builder +type AliasBuilder2 = builder + +fun AliasBuilder1.myBAlias(self): int1 { return 0 } +fun AliasBuilder2.myBAlias(self): int2 { return 0 } +fun builder?.myBAlias(self): int3 { return 0 } +fun T.myBAlias(self): int4 { return 0 } +fun U?.myBAlias(self): int5 { return 0 } + +fun test6(b: builder, b1: AliasBuilder1, b2: AliasBuilder2, b1n: AliasBuilder1?, b2n: AliasBuilder2?) { + __expect_type(b1.myBAlias(), "int1"); + __expect_type(b2.myBAlias(), "int2"); + __expect_type(b1n.myBAlias(), "int3"); + __expect_type((b2n as builder?).myBAlias(), "int3"); + + __expect_type((b as builder|slice).myBAlias(), "int4"); + __expect_type(0.myBAlias(), "int4"); + __expect_type((0 as int|int16).myBAlias(), "int4"); + __expect_type(b2n.myBAlias(), "int3"); + __expect_type((null as builder|slice?).myBAlias(), "int5"); + __expect_type((0 as int?).myBAlias(), "int5"); +} + +type AssetBuilder = builder +type BalanceBuilder = builder + +fun AssetBuilder.validate(self): int1 { return 0 } +fun BalanceBuilder.validate(self): int2 { return 0 } +fun Wrapper.validate(self): int3 { return 0 } +fun Wrapper.validate(self): int4 { return 0 } + +fun test7() { + var a: AssetBuilder = beginCell(); + var b: BalanceBuilder = beginCell(); + + __expect_type(a.validate(), "int1"); + __expect_type(b.validate(), "int2"); + __expect_type(Wrapper{item:a}.validate(), "int3"); + __expect_type(Wrapper{item:b}.validate(), "int4"); +} + +fun (slice|TT).supposeMe(self): TT { if (self is slice) { throw 123 } return self } +fun T.supposeMe(self): int2 { return 0 } +fun int|slice|null.supposeMe(self): int3 { return 0 } + +fun test8(a: cell|slice, b: cell|()|slice|builder, c:slice, d:int|slice, e:null|(slice|int), f: int?, g:cell) { + __expect_type(a.supposeMe(), "cell"); + __expect_type(b.supposeMe(), "cell | () | builder"); + __expect_type(c.supposeMe(), "slice"); + __expect_type(d.supposeMe(), "int"); + __expect_type(e.supposeMe(), "int3"); + __expect_type(f.supposeMe(), "int2"); + __expect_type(g.supposeMe(), "int2"); +} + +fun MyPair.mySize(self): int1 { return 0 } +fun MyPair.mySize(self): int2 { return 0 } +fun MyPair>.mySize(self): int3 { return 0 } +fun MyPair>.mySize(self): int4 { return 0 } +fun MyPair>.mySize(self): int5 { return 0 } + +fun test9( + m1: MyPair, m2: MyPair, + m3: MyPair>, m4: MyPair>, + m5: MyPair>, m6: MyPair>, + m7: MyPair>, m8: MyPair>, + m9: MyPair?>, m10: MyPair?>, + m11: MyPair>, m12: MyPair>, +) { + __expect_type(m1.mySize(), "int1"); + __expect_type(m2.mySize(), "int2"); + __expect_type(m3.mySize(), "int3"); + __expect_type(m4.mySize(), "int3"); + __expect_type(m5.mySize(), "int4"); + __expect_type(m6.mySize(), "int4"); + __expect_type(m7.mySize(), "int5"); + __expect_type(m8.mySize(), "int5"); + __expect_type(m9.mySize(), "int2"); + __expect_type(m10.mySize(), "int1"); + __expect_type(m11.mySize(), "int4"); + __expect_type(m12.mySize(), "int4"); +} + +fun MyPair>.h1(self): int1 { return 0 } +fun MyPair>.h1(self): int2 { return 0 } +fun MyPair>.h1(self): int3 { return 0 } +fun MyPair>.h1(self): int4 { return 0 } + +fun test10() { + __expect_type(MyPair>{}.h1(), "int1"); + __expect_type(MyPair>{}.h1(), "int2"); + __expect_type(MyPair>{}.h1(), "int3"); + __expect_type(MyPair>{}.h1(), "int4"); + __expect_type(MyPair>{}.h1(), "int4"); +} + +fun (int, U).p(self): int1 { return 0 } +fun (T, U).p(self): T { return self.0 } +fun T.p(self): int3 { return 0 } + +fun test11() { + __expect_type((5, "").p(), "int1"); + __expect_type((5 as int8, "").p(), "int8"); + __expect_type(("", "").p(), "slice"); + __expect_type((1, 2, 3).p(), "int3"); +} + +fun Wrapper.w(self): int1 { return 0 } +fun Wrapper.w(self): int2 { return 0 } + +fun test12() { + __expect_type(Wrapper{item:0}.w(), "int1"); + __expect_type(Wrapper{item:""}.w(), "int1"); + __expect_type(Wrapper>{item:{item:0}}.w(), "int2"); + // here the compiler finds T=slice resulting in Wrapper (it's correct) + // and chooses (1) due to a deeper shape + __expect_type(Wrapper{item:""}.w(), "int1"); +} + +type MyCell = cell +type MInt8 = int8 +type MInt32 = int32 + +fun MyCell.makeIterator(self): MyIterator { + return MyIterator.new(self) +} + +struct MyIterator { + data: slice +} + +fun MyIterator.new(c: cell): MyIterator { + return { data: c.beginParse() } +} + +fun MyIterator.empty(self): bool { + return self.data.isEmpty() +} + +fun MyIterator.next(mutate self): T { + return self.data.loadAny() +} + +fun MyCell.iterateAndAppend(self, mutate t: tuple) { + var it = self.makeIterator(); + while (!it.empty()) { + t.push(it.next()) + } +} + +fun MyCell.iterateAndAppend(self, mutate t: tuple) { + t.push(-777); +} + +@method_id(113) +fun test13() { + var c = beginCell().storeSlice(stringHexToSlice("0102")).endCell(); + var t = createEmptyTuple(); + (c as MyCell).iterateAndAppend(mutate t); + (c as MyCell).iterateAndAppend(mutate t); + (c as MyCell).iterateAndAppend(mutate t); + (c as MyCell).iterateAndAppend(mutate t); + return t; // [ 1 2 258 1 2 -777 ] +} + + +fun map.mySize(self): int1 { return 0 } +fun map.mySize(self): int2 { return 0 } +fun map.mySize(self): int3 { return 0 } +fun map>.mySize(self): int4 { return 0 } +fun map>.mySize(self): int5 { return 0 } + +type MInt16 = int16 +type MInt16ov = MInt16 +type MMap32 = map> +type MMap8Of32 = map + +fun test14( + m1: map>, m2: map>, + m3: map>, m4: map, m5: MMap8Of32, + m6: map?>, + m7: map, m8: map, + m9: map, +) { + __expect_type(m1.mySize(), "int5"); + __expect_type(m2.mySize(), "int5"); + __expect_type(m3.mySize(), "int4"); + __expect_type(m4.mySize(), "int4"); + __expect_type(m5.mySize(), "int4"); + __expect_type(m6.mySize(), "int3"); + __expect_type(m7.mySize(), "int2"); + __expect_type(m8.mySize(), "int2"); + __expect_type(m9.mySize(), "int1"); +} + +fun main() { + // besides these positive tests, there are many negative resulting in "ambiguous call" + return 0 +} + +/** +@testcase | 0 | | 0 +@testcase | 113 | | [ 1 2 258 1 2 -777 ] + */ diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 9c9acf821..823940319 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -7,6 +7,11 @@ struct Point { y: int32; } +struct TwoU { + a: uint8 + b: uint8 +} + @method_id(101) fun test1(value: int) { var t: JustInt32 = { value }; @@ -29,7 +34,7 @@ fun test1(value: int) { fun test2() { var t: JustInt32 = { value: 10 }; var c = t.toCell(); - var s = c.tvmCell.beginParse(); + var s = c.beginParse(); __expect_type(s.skipAny(), "slice"); s.assertEnd(); return true; @@ -84,6 +89,15 @@ fun test7(s: slice) { return (6, s.remainingBitsCount()); } +@method_id(108) +fun test8(): cell | int { + try { + return TwoU{a:1<<10, b:0}.toCell() + } catch (ex) { + return ex + } +} + fun main(c: cell) { c as Cell; (c as Cell) as cell; @@ -103,6 +117,7 @@ fun main(c: cell) { @testcase | 107 | x{09332} | 4 12 @testcase | 107 | x{2} | 5 1 @testcase | 107 | x{0234} | 6 16 +@testcase | 108 | | 5 1 @fif_codegen """ diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index cb01fbfeb..7bf0f247c 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -406,6 +406,26 @@ struct WithTwoRestFields { rest2: RemainingBitsAndRefs; } +struct WithMaps { + m1: map + m2: map +} + +struct WithNullableMaps { + m1: map? + m2: map? + m3: map? + m4: map | int32 +} + +enum EStoredAsInt8 { M100 = -100, Z = 0, P100 = 100 } +enum EStoredAsUint1 { ZERO, ONE } + +struct WithEnums { + e1: EStoredAsInt8 + e2: EStoredAsUint1 + rem: uint7 = 0 +} // --------------------------------------------- @@ -448,7 +468,7 @@ fun test_TwoInts32And64() { @method_id(205) fun test_TwoInts32AndRef64() { - run({ op: 123, query_id_ref: { tvmCell: beginCell().storeUint(255,64).endCell() } }, stringHexToSlice("0000007B").appendRef(stringHexToSlice("00000000000000FF"))); + run({ op: 123, query_id_ref: (255 as uint64).toCell() }, stringHexToSlice("0000007B").appendRef(stringHexToSlice("00000000000000FF"))); return true; } @@ -607,7 +627,7 @@ fun test_DifferentMix2() { assert(iae1.query_id_maybe_ref == null, 400); assert_slice_is_44_and_ref45(r1.rest); - run({ iae: { tvmCell: IntAndEither32OrRef64{ op: 778, i32orRef: { tvmCell: Inner2{ i64_in_ref: 9919992 }.toCell() }, query_id_maybe_ref: Inner1{ query_id_ref: 889477 }.toCell() }.toCell() }, tic: { op: 123, amount: 500000 }, rest: beginCell().endCell().beginParse() }, + run({ iae: IntAndEither32OrRef64{ op: 778, i32orRef: Inner2{ i64_in_ref: 9919992 }.toCell(), query_id_maybe_ref: Inner1{ query_id_ref: 889477 }.toCell() }.toCell(), tic: { op: 123, amount: 500000 }, rest: beginCell().endCell().beginParse() }, stringHexToSlice("0000007B307A120").appendRef(stringHexToSlice("0000030AE_").appendRef(stringHexToSlice("0000000000975DF8")).appendRef(stringHexToSlice("00000000000D9285")))); var r2 = DifferentMix2.fromSlice(stringHexToSlice("0000007B307A120").appendRef(stringHexToSlice("0000030AE_").appendRef(stringHexToSlice("0000000000975DF8")).appendRef(stringHexToSlice("00000000000D9285")))); val iae2 = r2.iae.load(); @@ -716,6 +736,60 @@ fun test_EdgeCaseIntegers() { return edge.toCell().hash() == manual.endCell().hash() } +@method_id(230) +fun test_serializeMaps() { + var ( + m1: map, + m2: map, + ) = (createEmptyMap(), createEmptyMap()); + m1.set(1, ""); + m1.set(2, stringHexToSlice("01").appendRef(stringHexToSlice("02"))); + m2.set({value: 3}, (3, 30)); + m2.set({value: 4}, (4, 40)); + + var w = (WithMaps { m1, m2 }).toCell().load(); + w.m2.delete({value: 4}); + val d3 = w.m2.deleteAndGetDeleted({value: 3}); + var r = w.m1.findFirst(); + var kl = (0, 0); + while (r.isFound) { + kl = r.loadValue().remainingBitsAndRefsCount(); + r = w.m1.iterateNext(r); + } + return (w.m1.isEmpty(), w.m2.isEmpty(), d3.isFound, d3.loadValue(), kl); +} + +@method_id(231) +fun test_nullableMaps() { + var o: WithNullableMaps = { + m1: createEmptyMap(), + m2: createEmptyMap(), + m3: null, + m4: createEmptyMap(), + }; + if (o.m2 != null && o.m4 is map) { + o.m2.set(1, 10); + o.m4.set(1, 10); + } + var c = o.toCell(); + var s = c.beginParse(); + var (bits, refs) = s.remainingBitsAndRefsCount(); + assert(bits == 2+2+1+2) throw 123; + assert(refs == 0+1+0+1) throw 234; + + o = c.load(); + return (o.m1 == null, o.m1!.isEmpty(), o.m2 == null, o.m2!.isEmpty(), o.m3 == null, o.m4 is int32); +} + +@method_id(232) +fun test_SimpleEnums() { + run({e1: EStoredAsInt8.M100, e2: EStoredAsUint1.ZERO}, stringHexToSlice("9c00")); + run({e1: EStoredAsInt8.Z, e2: EStoredAsUint1.ZERO}, stringHexToSlice("0000")); + run({e1: EStoredAsInt8.P100, e2: EStoredAsUint1.ONE, rem: 1}, stringHexToSlice("6481")); + return -1; +} + + fun main() { var t: JustInt32 = { value: 10 }; var c = t.toCell(); @@ -737,18 +811,18 @@ fun main() { @testcase | 208 | | -1 @testcase | 209 | | -1 @testcase | 210 | | 123 555 46 (null) -@testcase | 211 | | (null) typeid-4 123 -@testcase | 212 | | (null) typeid-5 123 +@testcase | 211 | | (null) typeid-3 123 +@testcase | 212 | | (null) typeid-4 123 @testcase | 213 | | -1 @testcase | 214 | | -1 @testcase | 215 | | -1 @testcase | 216 | | -1 @testcase | 217 | | -1 @testcase | 218 | | -1 -@testcase | 219 | | 44 (null) (null) 46 typeid-15 +@testcase | 219 | | 44 (null) (null) 46 typeid-11 @testcase | 220 | | 0 0 (null) 99 typeid-8 typeid-9 99 1234 (null) 889129 14 @testcase | 221 | | -1 -@testcase | 222 | | 510 567 9392843922 typeid-18 81923 81923 typeid-19 777 0 -1 (null) (null) 0 100000 100000 typeid-19 +@testcase | 222 | | 510 567 9392843922 typeid-13 81923 81923 typeid-14 777 0 -1 (null) (null) 0 100000 100000 typeid-14 @testcase | 223 | | 10 55 (null) @testcase | 224 | | 60 50000000 5 9 123 -1 @testcase | 225 | | 5 40 65535 8 50000000 @@ -756,4 +830,7 @@ fun main() { @testcase | 227 | | 16 @testcase | 228 | | -1 @testcase | 229 | | -1 +@testcase | 230 | | 0 -1 -1 3 30 8 1 +@testcase | 231 | | 0 -1 0 0 -1 0 +@testcase | 232 | | -1 */ diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk index 793f714b9..71331464f 100644 --- a/tolk-tester/tests/pack-unpack-3.tolk +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -248,8 +248,8 @@ fun test_MsgExternal1() { run(SayHiAndGoodbye{ dest_addr: null, body: BodyPayload2 { master_id: 10, owner_address: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") } }, stringHexToSlice("892150000000000000000000000000000000000000000000000000000000000000000002_")); run(SayHiAndGoodbye{ dest_addr: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), body: BodyPayload1 { should_forward: false, n_times: 85, content: generateSlice_44_with_ref45() } }, stringHexToSlice("89C0000000000000000000000000000000000000000000000000000000000000000002000000550000002C").appendRef(stringHexToSlice("0000002D"))); run(SayHiAndGoodbye{ dest_addr: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), body: BodyPayload2 { master_id: -5, owner_address: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi") } }, stringHexToSlice("89CFF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E7EE7F9474019ED8A33E5675458AC787B0F3416F037452CA3A82648278FB81E95F26CF4_")); - run(SayStoreInChain{ in_masterchain: true, contents: { tvmCell: BodyPayload1{ should_forward: true, n_times: 20, content: generateSlice_44_with_ref45() }.toCell()} }, stringHexToSlice("0013C_").appendRef(stringHexToSlice("3000000140000002C").appendRef(stringHexToSlice("0000002D")))); - run(SayStoreInChain{ in_masterchain: false, contents: { tvmCell: BodyPayload2{ master_id: 37, owner_address: address(("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c")) }.toCell()} }, stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); + run(SayStoreInChain{ in_masterchain: true, contents: BodyPayload1{ should_forward: true, n_times: 20, content: generateSlice_44_with_ref45() }.toCell() as cell as Cell }, stringHexToSlice("0013C_").appendRef(stringHexToSlice("3000000140000002C").appendRef(stringHexToSlice("0000002D")))); + run(SayStoreInChain{ in_masterchain: false, contents: BodyPayload2{ master_id: 37, owner_address: address(("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c")) }.toCell() as cell as Cell }, stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); val o = MsgExternal1.fromSlice(stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); assert(o is SayStoreInChain, 400); @@ -260,15 +260,15 @@ fun test_MsgExternal1() { @method_id(205) fun test_MsgTransfer() { run({ params: EitherLeft { value: TransferParams1 { dest_int: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), amount: 80000000, dest_ext: makeExternalAddress(1234,80) } } }, stringHexToSlice("FB3701FF3CA4FF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E404C4B4004A0000000000000000009A5_")); - run({ params: EitherLeft { value: TransferParams2 { intVector: (123, 1234567890123456, 1234567890123456), needs_more: {tvmCell: beginCell().storeBool(true).endCell()} } } }, stringHexToSlice("FB3701FF48000003DDC118B54F22AEB0000118B54F22AEB02_").appendRef(stringHexToSlice("C_"))); - run({ params: EitherRight { value: { tvmCell: TransferParams1{ dest_int: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), amount: 80000000, dest_ext: makeExternalAddress(1234,70) }.toCell()} } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("7948000000000000000000000000000000000000000000000000000000000000000000809896800918000000000000004D2"))); - run({ params: EitherRight { value: { tvmCell: TransferParams2{ intVector: (123, null, 0), needs_more: {tvmCell: beginCell().storeBool(false).endCell()} }.toCell()} } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); + run({ params: EitherLeft { value: TransferParams2 { intVector: (123, 1234567890123456, 1234567890123456), needs_more: true.toCell() } } }, stringHexToSlice("FB3701FF48000003DDC118B54F22AEB0000118B54F22AEB02_").appendRef(stringHexToSlice("C_"))); + run({ params: EitherRight { value: TransferParams1{ dest_int: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), amount: 80000000, dest_ext: makeExternalAddress(1234,70) }.toCell() as cell as Cell } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("7948000000000000000000000000000000000000000000000000000000000000000000809896800918000000000000004D2"))); + run({ params: EitherRight { value: TransferParams2{ intVector: (123, null, 0), needs_more: false.toCell() }.toCell() as cell as Cell } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); val o = MsgTransfer.fromSlice(stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); return ( o.params is EitherLeft, o.params is EitherRight, - o.params is EitherRight && TransferParams.fromCell(o.params.value.tvmCell) is TransferParams1, - o.params is EitherRight && TransferParams.fromCell(o.params.value.tvmCell) is TransferParams2, + o.params is EitherRight && TransferParams.fromCell(o.params.value) is TransferParams1, + o.params is EitherRight && TransferParams.fromCell(o.params.value) is TransferParams2, ); } diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 97b740b48..05a9ee1e6 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -10,29 +10,34 @@ struct JustMaybeInt32 { value: MaybeInt32; } +fun T.estimate(): [int, int, int, int] { + // convert a tensor to a tuple for better output readability + val (minBits, maxBits, minRefs, maxRefs) = T.estimatePackSize(); + return [minBits, maxBits, minRefs, maxRefs]; +} @method_id(101) fun test1() { return ( - int32.estimatePackSize(), - uint64.estimatePackSize(), - int1.estimatePackSize(), - bool.estimatePackSize(), - RemainingBitsAndRefs.estimatePackSize(), - coins.estimatePackSize(), - bits6.estimatePackSize(), - bytes8.estimatePackSize(), - varint32.estimatePackSize(), + int32.estimate(), + uint64.estimate(), + int1.estimate(), + bool.estimate(), + RemainingBitsAndRefs.estimate(), + coins.estimate(), + bits6.estimate(), + bytes8.estimate(), + varint32.estimate(), ); } @method_id(102) fun test2() { return ( - JustInt32.estimatePackSize(), - JustMaybeInt32.estimatePackSize(), - MaybeInt32.estimatePackSize(), - Int16Or32.estimatePackSize(), + JustInt32.estimate(), + JustMaybeInt32.estimate(), + MaybeInt32.estimate(), + Int16Or32.estimate(), ); } @@ -51,7 +56,7 @@ struct Test3_2 { @method_id(103) fun test3() { - return (Test3_1.estimatePackSize(), Test3_2.estimatePackSize()); + return (Test3_1.estimate(), Test3_2.estimate()); } struct Test4_1 { @@ -71,7 +76,7 @@ struct Test4_3 { @method_id(104) fun test4() { - return (Test4_1.estimatePackSize(), Test4_2.estimatePackSize(), Test4_3.estimatePackSize()); + return (Test4_1.estimate(), Test4_2.estimate(), Test4_3.estimate()); } struct Test5_1 { @@ -99,7 +104,7 @@ struct Test5_3 { @method_id(105) fun test5() { - return (Test5_1.estimatePackSize(), Test5_2.estimatePackSize(), Test5_3.estimatePackSize()); + return (Test5_1.estimate(), Test5_2.estimate(), Test5_3.estimate()); } struct(0x00112233) Test6_1 { @@ -115,7 +120,7 @@ type Test6_or = Test6_1 | Test6_2; @method_id(106) fun test6() { - return (Test6_1.estimatePackSize(), Test6_2.estimatePackSize(), Test6_or.estimatePackSize()); + return (Test6_1.estimate(), Test6_2.estimate(), Test6_or.estimate()); } struct(0x1020) Test7_1; @@ -127,7 +132,7 @@ type Test7_or = | Test7_1 | Test7_2 | Test7_3 @method_id(107) fun test7() { assert((Test7_1{} as Test7_or).toCell().beginParse().remainingBitsCount() == Test7_or.estimatePackSize().0, 400); - return (Test7_1.estimatePackSize(), Test7_or.estimatePackSize()); + return (Test7_1.estimate(), Test7_or.estimate()); } struct(0x10) Inner8_1 { @@ -151,7 +156,7 @@ struct Test8 { @method_id(108) fun test8() { - return (Inner8_1.estimatePackSize(), Inner8_2.estimatePackSize(), Test8.estimatePackSize()); + return (Inner8_1.estimate(), Inner8_2.estimate(), Test8.estimate()); } struct Test9_bits2 { f: bits2; } @@ -164,7 +169,7 @@ type Test9_f4 = bits1 | Test9_bits2 | bits3 | Test9_bits4?; // auto-generate @method_id(109) fun test9() { - return (Test9_f1.estimatePackSize(), Test9_f2.estimatePackSize(), Test9_f3.estimatePackSize(), Test9_f4.estimatePackSize()); + return (Test9_f1.estimate(), Test9_f2.estimate(), Test9_f3.estimate(), Test9_f4.estimate()); } struct Test10_1 { @@ -176,7 +181,7 @@ type Test10_2 = (Test10_1, bool?, RemainingBitsAndRefs); @method_id(110) fun test10() { - return (Test10_1.estimatePackSize(), Test10_2.estimatePackSize()); + return (Test10_1.estimate(), Test10_2.estimate()); } struct Test11_1 { @@ -191,7 +196,7 @@ struct Test11_2 { @method_id(111) fun test11() { - return (Test11_1.estimatePackSize(), Test11_2.estimatePackSize()); + return (Test11_1.estimate(), Test11_2.estimate()); } @method_id(120) @@ -200,8 +205,25 @@ fun test20() { CellInner8_1.getDeclaredPackPrefixLen(), CellInner8_1.getDeclaredPackPrefix()); } +enum Color { Red, Green, Blue } +enum EInt9 { P133 = 133, M1 = -1 } +enum EUint6 { P63 = 63, P30 = 30, P1 = 1 } +enum EUint1 { A, B } +enum EInt1 { A = -1, B } +enum EUint123 { Z, V5 = 5, E123 = 1<<122, E123_2 = (1<<122) + (1<<121) } +type EUint123Alias = EUint123 + +@method_id(121) +fun test21() { + return ( + Color.estimate(), EInt9.estimate(), EUint6.estimate(), + EUint1.estimate(), EInt1.estimate(), EInt1.estimate(), + EUint123.estimate(), EUint123Alias.estimate(), + ) +} + fun main() { - __expect_type(int8.estimatePackSize(), "[int, int, int, int]"); + __expect_type(int8.estimatePackSize(), "(int, int, int, int)"); } /** @@ -217,4 +239,5 @@ fun main() { @testcase | 110 | | [ 32 9999 0 4 ] [ 33 9999 0 8 ] @testcase | 111 | | [ 1023 1023 0 1 ] [ 0 0 2 2 ] @testcase | 120 | | 16 4128 1 1 +@testcase | 121 | | [ 2 2 0 0 ] [ 9 9 0 0 ] [ 6 6 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 123 123 0 0 ] [ 123 123 0 0 ] */ diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 364d026a2..4a261869c 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -26,9 +26,7 @@ fun test2() { @method_id(103) fun test3() { - var cc: Cell = { - tvmCell: beginCell().storeSlice(stringHexToSlice("0109ab")).endCell() - }; + var cc = beginCell().storeSlice(stringHexToSlice("0109ab")).endCell() as Cell; return cc.load({assertEndAfterReading: false}); } @@ -146,14 +144,14 @@ RETALT """ IF:<{ DROP - 139 PUSHINT + 129 PUSHINT }>ELSE<{ x{03} SDBEGINSQ NIP IFNOTJMP:<{ 104 THROW }> - 140 PUSHINT + 130 PUSHINT }> """ @@ -201,7 +199,7 @@ IF:<{ x{01} SDBEGINSQ IF:<{ DROP - 139 PUSHINT + 129 PUSHINT }>ELSE<{ x{03} SDBEGINSQ NIP @@ -209,9 +207,9 @@ IF:<{ 16 PUSHPOW2DEC THROWANY }> - 140 PUSHINT + 130 PUSHINT }> - 139 PUSHINT + 129 PUSHINT EQUAL }> """ diff --git a/tolk-tester/tests/pack-unpack-7.tolk b/tolk-tester/tests/pack-unpack-7.tolk index 60e9c2b5c..0a11b378a 100644 --- a/tolk-tester/tests/pack-unpack-7.tolk +++ b/tolk-tester/tests/pack-unpack-7.tolk @@ -100,6 +100,22 @@ fun Tensor3Skipping1.packToBuilder(self, mutate b: builder) { b.storeUint(self.0, 8).storeUint(self.2, 8); } +enum Color { + Red, + Green, + Blue, +} + +type UnsafeColor = Color + +fun UnsafeColor.unpackFromSlice(mutate s: slice) { + return s.loadUint(2) as Color +} + +fun UnsafeColor.packToBuilder(self, mutate b: builder) { + b.storeUint(self as int, 2) +} + @method_id(101) fun test1() { @@ -155,6 +171,15 @@ fun test7() { return t.toCell().load(); } +@method_id(108) +fun test8(c: UnsafeColor) { + var way1 = beginCell().storeAny(c).endCell(); + var way2 = c.toCell(); + __expect_type(way2, "Cell"); + assert (way1.hash() == way2.hash()) throw 123; + return way2.load(); +} + fun main() { __expect_type("" as TelegramString, "TelegramString"); } @@ -169,6 +194,8 @@ fun main() { @testcase | 105 | | 123 1 @testcase | 106 | | 6 2 255 0 8 9 @testcase | 107 | | 1 0 3 +@testcase | 108 | 0 | 0 +@testcase | 108 | 3 | 3 @fif_codegen """ diff --git a/tolk-tester/tests/pack-unpack-8.tolk b/tolk-tester/tests/pack-unpack-8.tolk new file mode 100644 index 000000000..a146996a1 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-8.tolk @@ -0,0 +1,215 @@ +enum EFits2Bits { ZERO, ONE, TWO } + +fun EFits2Bits.sliceWithZero(): slice asm "b{00} PUSHSLICE" +fun EFits2Bits.sliceWithThree(): slice asm "b{11} PUSHSLICE" + +enum EStartFrom1 { + ONE = 1 + TWO + THREE +} + +enum EFillsAllUint3 { + p0, p1, p2, p3, p4, p5, p6, p7 +} + +enum EStartFromM2 { + M2 = -2, + M1, ZERO, P1, P2, P3 = 3, +} + +enum EFits8Bits { + E0 = 0; + E110 = 110; + E220 = 220; +} + +const MAX_UINT = 115792089237316195423570985008687907853269984665640564039457584007913129639935 + +enum EMinMax { + MIN_INT = -115792089237316195423570985008687907853269984665640564039457584007913129639935 - 1, + MAX_INT = +115792089237316195423570985008687907853269984665640564039457584007913129639935, +} + +enum E0Max { + ZERO, + MAX_INT = +115792089237316195423570985008687907853269984665640564039457584007913129639935, +} + +struct WithEnumsUnion { + u: EFits8Bits | EStartFromM2 +} + +enum Role: int8 { + Admin, User, +} + +fun (Role, Role).decode(mutate self, s: slice) { + self = s.loadAny() +} + +enum EncodedVari : varint16 { + ONE = 1, + TWO = 2, + MANY = 1<<100, +} + + +@method_id(101) +fun test1() { + var e = EFits2Bits.fromSlice(EFits2Bits.sliceWithZero()); + return (e == EFits2Bits.ZERO, e == EFits2Bits.TWO); +} + +@method_id(102) +fun test2(c: EFits2Bits) { + try { + return (c.toCell().beginParse().loadAny() == c) as int + } catch (ex) { + return ex * 10 + } +} + +@method_id(103) +fun test3() { + return EStartFrom1.fromCell(EStartFrom1.ONE.toCell()) +} + +@method_id(104) +fun test4(v: EFillsAllUint3) { + return EFillsAllUint3.fromCell(v.toCell()) +} + +@method_id(105) +fun test5(s: slice) { + return EStartFromM2.fromSlice(s, {assertEndAfterReading: false}) +} + +@method_id(106) +fun test6() { + try { + return (4 as EFits2Bits).toCell().beginParse().loadUint(1) + } catch (ex) { + return ex + } +} + +@method_id(107) +fun test7(v: EFits8Bits) { + try { return EFits8Bits.fromCell(v.toCell()) as int } + catch (ex) { return ex * 1000 } +} + +@method_id(108) +fun test8() { + var s_min = beginCell().storeInt(-MAX_UINT - 1, 257).endCell(); + var s_max = beginCell().storeInt(+MAX_UINT, 257).endCell(); + return ( + EMinMax.fromCell(s_min) == EMinMax.MIN_INT, EMinMax.fromCell(s_max) == EMinMax.MAX_INT, + EMinMax.fromCell(s_max) == EMinMax.MIN_INT, EMinMax.fromCell(s_min) == EMinMax.MAX_INT, + ) +} + +@method_id(109) +fun test9() { + var s_min = beginCell().storeUint(0, 256).endCell(); + var s_max = beginCell().storeUint(MAX_UINT, 256).endCell(); + return ( + E0Max.fromCell(s_min) == E0Max.ZERO, E0Max.fromCell(s_max) == E0Max.MAX_INT, + ) +} + +@method_id(110) +fun test10() { + var w1 = WithEnumsUnion{u: EFits8Bits.E110}.toCell().load(); + var w2 = WithEnumsUnion{u: (220 as EFits8Bits)}.toCell().load(); + var w3 = WithEnumsUnion{u: ([0] as tuple).get(0)}.toCell().load(); + var w4 = WithEnumsUnion{u: EStartFromM2.P3}.toCell().load(); + return ( + w1.u is EFits8Bits && w1.u == EFits8Bits.E110, + w2.u is EFits8Bits && w2.u == EFits8Bits.E220, + w3.u is EStartFromM2 && w3.u == EStartFromM2.ZERO, + w4.u is EStartFromM2 && w4.u == EStartFromM2.P3, + w4.u is EFits8Bits && w4.u == EFits8Bits.E220, + ) +} + +@method_id(111) +fun test11() { + var (r1, r2) = (-1 as Role, -1 as Role); + val before = (r1 == Role.Admin, r2 == Role.Admin); + (r1, r2).decode(stringHexToSlice("0001")); + return (before, r1, r2, r1 == Role.Admin, r2 == Role.Admin, r1 == Role.User, r2 == Role.User) +} + +@method_id(112) +fun test12() { + var c3_one = EncodedVari.fromSlice(stringHexToSlice("1018_")); + var c3_many = EncodedVari.fromSlice(stringHexToSlice("d100000000000000000000000008_")); + return (c3_one, c3_one == EncodedVari.MANY, c3_many == EncodedVari.MANY); +} + +fun main() { + +} + +/** +@testcase | 101 | | -1 0 +@testcase | 102 | 1 | -1 +@testcase | 102 | 3 | 50 +@testcase | 103 | | 1 +@testcase | 104 | 3 | 3 +@testcase | 104 | 7 | 7 +@testcase | 105 | x{d} | -2 +@testcase | 105 | x{7} | 3 +@testcase | 106 | | 5 +@testcase | 107 | 0 | 0 +@testcase | 107 | 110 | 110 +@testcase | 107 | 220 | 220 +@testcase | 107 | -50 | 5000 +@testcase | 107 | 1 | 5000 +@testcase | 107 | 222 | 5000 +@testcase | 108 | | -1 -1 0 0 +@testcase | 109 | | -1 -1 +@testcase | 110 | | -1 -1 -1 -1 0 +@testcase | 111 | | 0 0 0 1 -1 0 0 -1 +@testcase | 112 | | 1 0 -1 + +@fif_codegen +""" + test1() PROC:<{ + b{00} PUSHSLICE + 2 LDU + OVER + 2 GTINT + 5 THROWIF + ENDS +""" + +@fif_codegen +""" + test4() PROC:<{ // v + NEWC // v b + 3 STU // b + ENDC // '4 + CTOS // s + 3 LDU // '9 s + ENDS // '9 + }> +""" + +@fif_codegen +""" + test5() PROC:<{ + // s + 3 PLDI + DUP + -2 LESSINT + 5 THROWIF + DUP + 3 GTINT + 5 THROWIF + }> +""" + + */ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 0079871a6..b4613655c 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -30,6 +30,7 @@ fun check0(addr: address) { assert (addr.getWorkchain() == 0) throw 111; } +@method_id(90) fun codegenAddrEq(a: address, b: address) { if (a == b) { return 1; } if (a != b) { return 2; } @@ -141,12 +142,14 @@ fun main() { (cc1 as slice).bitsEqual(((cc2 as slice) as address) as slice), createAddressNone() == cc1, createAddressNone() == address.createNone(), - check0(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")) + check0(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")), + cc1 == cc1 as bits267 as address, + cc1 == cc1 as slice as bits999 as address, ); } /** -@testcase | 0 | | -1 0 -1 0 -1 +@testcase | 0 | | -1 0 -1 0 -1 -1 -1 @fif_codegen """ diff --git a/tolk-tester/tests/remove-unused-functions.tolk b/tolk-tester/tests/remove-unused-functions.tolk index a6b910009..71f71d762 100644 --- a/tolk-tester/tests/remove-unused-functions.tolk +++ b/tolk-tester/tests/remove-unused-functions.tolk @@ -28,8 +28,6 @@ fun main(): (int, int, int) { } /** -@experimental_options remove-unused-functions - @testcase | 0 | | 3 10 20 @fif_codegen DECLPROC used_as_noncall1() diff --git a/tolk-tester/tests/send-msg-1.tolk b/tolk-tester/tests/send-msg-1.tolk index 3a889dcf4..c15b2d356 100644 --- a/tolk-tester/tests/send-msg-1.tolk +++ b/tolk-tester/tests/send-msg-1.tolk @@ -170,13 +170,13 @@ fun test4(value: coins) { @noinline fun test5_manual() { - var ec_dict = createEmptyDict(); - ec_dict.iDictSet(32, 1, "ec1"); + var ec_dict: ExtraCurrenciesMap = createEmptyMap(); + ec_dict.set(1, 100); return beginCell() .storeUint(0x18, 6) // bounce .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) .storeCoins(123) // value.grams - .storeMaybeRef(ec_dict) // value.extra + .storeMaybeRef(ec_dict.toLowLevelDict()) // value.extra .storeUint(0, 0 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) // body: op + queryId .storeUint(0x12345678, 32) @@ -186,8 +186,8 @@ fun test5_manual() { @method_id(105) fun test5() { - var ec_dict = createEmptyDict(); - ec_dict.iDictSet(32, 1, "ec1"); + var ec_dict: ExtraCurrenciesMap = createEmptyMap(); + ec_dict.set(1, 100); val body: MyBody = { queryId: 800 }; var b = createMessage({ body, @@ -215,9 +215,9 @@ fun test6_manual(value: coins, ec_dict: dict) { @method_id(106) fun test6(value: coins, dictKey: int?) { - var ec_dict = createEmptyDict(); + var ec_dict: ExtraCurrenciesMap = createEmptyMap(); if (dictKey != null) { - ec_dict.iDictSet(32, dictKey, "ec1"); + ec_dict.set(dictKey, 100500); } val body: MyBody = { queryId: 800 }; var b = createMessage({ @@ -226,7 +226,7 @@ fun test6(value: coins, dictKey: int?) { value: (value, ec_dict), body, }); - assert(b.hash() == test6_manual(value, ec_dict).hash(), 106); + assert(b.hash() == test6_manual(value, ec_dict.toLowLevelDict()).hash(), 106); return b.hash(); } @@ -617,7 +617,7 @@ fun test21(queryId: uint64) { var b = createMessage({ bounce: false, dest: address("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), - value: (ton("0.1"), createEmptyDict()), + value: (ton("0.1"), createEmptyMap()), body, }); assert(b.hash() == test21_manual(body).hash(), 121); @@ -732,9 +732,9 @@ fun main() { @testcase | 102 | | 20192939504955637835708665496574868659039935190188156593169026529135727309085 @testcase | 103 | | 97654287980681401436727082042373998183264988661375688119718809500301413968039 @testcase | 104 | 600000 | 755636693689039391990782995030008885663781699175339033402019272057515711062 -@testcase | 105 | | 24816341499673835567887890014278904436471582322802948121781418622643959482495 +@testcase | 105 | | 24503969966890554363249789782591743702412487637374098978386746563683471749050 @testcase | 106 | 123 null | 104134380371273907780196393444164225714229635235007677971195651971972203592811 -@testcase | 106 | 456 8888 | 39337586036945718311402746340438400160817844833530971545330721291986281100430 +@testcase | 106 | 456 8888 | 6425272102977874346517989655071023150283043721608819726435980878339463285784 @testcase | 107 | 1000 | 55093441331748625324828489600632232039914212774002148634088483962817636598198 @testcase | 108 | 50000000 | 95023796475113775225029817428715936488418545169963429399979521091689824066088 @testcase | 109 | 0 | 55999621586681214992294941423256376619779969729861696464321825639854258502733 diff --git a/tolk-tester/tests/send-msg-4.tolk b/tolk-tester/tests/send-msg-4.tolk new file mode 100644 index 000000000..84706fd71 --- /dev/null +++ b/tolk-tester/tests/send-msg-4.tolk @@ -0,0 +1,60 @@ +struct LiquidityDepositWithInitData { + code: cell + data: cell +} + +type LiquidityDepositAddress = address + +type LiquidityDepositDestination = LiquidityDepositWithInitData | LiquidityDepositAddress + +struct (0x1b434676) AddLiquidityPartTon { + destination: LiquidityDepositDestination +} + +type AllowedMessages = AddLiquidityPartTon + +fun main() { + val msgCell = beginCell(). + storeUint(0x1b434676, 32) + .storeBool(true) // either right + .storeAddress(address("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4")) + .endCell(); + + val msg = lazy AllowedMessages.fromCell(msgCell); + + match (msg) { + AddLiquidityPartTon => { + // dynamic union: the compiler inserts lots of IFs at runtime to handle this + var destination: address | AutoDeployAddress; + + match (msg.destination) { + LiquidityDepositWithInitData => { + destination = { + stateInit: { code: msg.destination.code, data: msg.destination.data }, + }; + } + LiquidityDepositAddress => { + destination = msg.destination; + } + } + + val sendMsg = createMessage({ + bounce: false, + dest: destination, + value: 0, + }); + sendMsg.send(SEND_MODE_REGULAR); + } + } + + return 0; +} + +/** +@testcase | 0 | | 0 + +@fif_codegen ONE HASHEXT_SHA256 +@fif_codegen }>ELSE<{ +@fif_codegen HASHCU +@fif_codegen 256 STU + */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 90f1ae644..589bf6523 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -610,7 +610,7 @@ fun test55() { // in case of loops, type inferring is re-enterable // first iteration: x is int, eq instantiated // second (final) iteration: x is int?, eq instantiated - // (checked via codegen) + // (but since eq finally becomes unused, it's not generated to fift) eq55(x); __expect_type(x, "int?"); // types are checked (unlike generics instantiated) after inferring x = rand() ? 1 : null; @@ -716,8 +716,70 @@ fun test63() { __expect_type(r!.sub, "Point"); } +type BalanceDict = dict + +fun cell.myDepth(self): int1 { return 0 } +fun BalanceDict.myDepth(self): int3 { return 0 } +fun null.myDepth(self): int4 { return 0 } + +fun test64(d: dict, b: BalanceDict) { + match (d) { + null => { __expect_type(d.myDepth(), "int4") } + cell => { __expect_type(d.myDepth(), "int1") } + } + __expect_type(d, "dict"); + __expect_type(d.myDepth(), "int3"); + + match (b) { + null => { __expect_type(b.myDepth(), "int4") } + cell => { __expect_type(b.myDepth(), "int1") } + } + __expect_type(b, "BalanceDict"); + __expect_type(b.myDepth(), "int3"); + + if (d == null) { return; } + if (b != null) { return; } + + __expect_type(d.myDepth(), "int1"); + __expect_type(b.myDepth(), "int4"); + + if ([0].0) { + (b, d) = (createEmptyCell(), null) + } + __expect_type(b, "BalanceDict"); + __expect_type(d, "dict"); +} + +struct WithDict { + d: dict + b: BalanceDict +} + +fun test65(w: WithDict, t: (dict, BalanceDict)) { + match (w.d) { + null => { __expect_type(w.d, "null") } + cell => { __expect_type(w.d, "cell") } + } + if (w.b == null) {} + else { __expect_type(w.b, "cell") } + + match (t.1) { + null => { __expect_type(t.1.myDepth(), "int4") } + cell => { __expect_type(t.1.myDepth(), "int1") } + } + if (t.0 != null) { __expect_type(t.0, "cell") } + + __expect_type(w.d, "dict"); + __expect_type(w.b, "BalanceDict"); + __expect_type(t.0, "dict"); + __expect_type(t.1, "BalanceDict"); +} + fun main(x: int?): int { + // mark used to codegen them + test55; + return x == null ? -1 : x; } @@ -734,23 +796,23 @@ fun main(x: int?): int { @testcase | 139 | | 16 @testcase | 140 | 5 | 25 @testcase | 141 | | 1 2 -@testcase | 142 | | 5 3 (null) (null) 0 typeid-5 3 (null) (null) 0 +@testcase | 142 | | 5 3 (null) (null) 0 typeid-3 3 (null) (null) 0 @testcase | 143 | | 10 11 (null) 10 11 (null) (null) 0 @testcase | 144 | | 10 11 (null) 10 11 (null) (null) 0 -@testcase | 145 | | 5 3 (null) (null) 0 typeid-5 3 (null) (null) (null) (null) 0 3 (null) (null) 0 -@testcase | 146 | | 3 4 5 3 4 5 typeid-4 +@testcase | 145 | | 5 3 (null) (null) 0 typeid-3 3 (null) (null) (null) (null) 0 3 (null) (null) 0 +@testcase | 146 | | 3 4 5 3 4 5 typeid-2 @testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 @testcase | 158 | | 123 10 123 5 @testcase | 160 | | 101 109 @testcase | 161 | 9 9 | (null) (null) 0 (null) (null) -@testcase | 161 | 19 0 | 19 0 typeid-1 19 0 +@testcase | 161 | 19 0 | 19 0 typeid-4 19 0 @stderr warning: expression of type `int` can never be `null`, this condition is always true @stderr warning: unreachable code @stderr var t2 redef = getNullableInt()!; -@fif_codegen eq55() PROC:<{ -@fif_codegen eq55() PROC:<{ +@fif_codegen eq55() PROC:<{ +@fif_codegen_avoid eq55() PROC:<{ @fif_codegen """ diff --git a/tolk-tester/tests/some-tests-4.tolk b/tolk-tester/tests/some-tests-4.tolk new file mode 100644 index 000000000..74645e15e --- /dev/null +++ b/tolk-tester/tests/some-tests-4.tolk @@ -0,0 +1,110 @@ +const __precisionZeros = 1000000000000000000; // 18 zeros +const __base0 = 1000000043929018416; // 4x every year (3600 * (24 * 365 + 6) seconds) + +const __base1 = __base0 * __base0 / __precisionZeros; +const __base2 = __base1 * __base1 / __precisionZeros; +const __base3 = __base2 * __base2 / __precisionZeros; +const __base4 = __base3 * __base3 / __precisionZeros; +const __base5 = __base4 * __base4 / __precisionZeros; +const __base6 = __base5 * __base5 / __precisionZeros; +const __base7 = __base6 * __base6 / __precisionZeros; +const __base8 = __base7 * __base7 / __precisionZeros; +const __base9 = __base8 * __base8 / __precisionZeros; +const __base10 = __base9 * __base9 / __precisionZeros; +const __base11 = __base10 * __base10 / __precisionZeros; +const __base12 = __base11 * __base11 / __precisionZeros; +const __base13 = __base12 * __base12 / __precisionZeros; +const __base14 = __base13 * __base13 / __precisionZeros; +const __base15 = __base14 * __base14 / __precisionZeros; +const __base16 = __base15 * __base15 / __precisionZeros; +const __base17 = __base16 * __base16 / __precisionZeros; +const __base18 = __base17 * __base17 / __precisionZeros; +const __base19 = __base18 * __base18 / __precisionZeros; +const __base20 = __base19 * __base19 / __precisionZeros; +const __base21 = __base20 * __base20 / __precisionZeros; +const __base22 = __base21 * __base21 / __precisionZeros; +const __base23 = __base22 * __base22 / __precisionZeros; +const __base24 = __base23 * __base23 / __precisionZeros; +const __base25 = __base24 * __base24 / __precisionZeros; +const __base26 = __base25 * __base25 / __precisionZeros; +const __base27 = __base26 * __base26 / __precisionZeros; +const __base28 = __base27 * __base27 / __precisionZeros; +const __base29 = __base28 * __base28 / __precisionZeros; +const __base30 = __base29 * __base29 / __precisionZeros; + +const ERROR_EXP_TOO_LARGE = 10970; + +@method_id(101) +fun test1(value: int, exp: int): int { + assert (exp <= 0x7fffffff) throw ERROR_EXP_TOO_LARGE; + + if ((exp & 0x00000001) != 0) { value = mulDivFloor(value, __base0, __precisionZeros); } + if ((exp & 0x00000002) != 0) { value = mulDivFloor(value, __base1, __precisionZeros); } + if ((exp & 0x00000004) != 0) { value = mulDivFloor(value, __base2, __precisionZeros); } + if ((exp & 0x00000008) != 0) { value = mulDivFloor(value, __base3, __precisionZeros); } + + if ((exp & 0x00000010) != 0) { value = mulDivFloor(value, __base4, __precisionZeros); } + if ((exp & 0x00000020) != 0) { value = mulDivFloor(value, __base5, __precisionZeros); } + if ((exp & 0x00000040) != 0) { value = mulDivFloor(value, __base6, __precisionZeros); } + if ((exp & 0x00000080) != 0) { value = mulDivFloor(value, __base7, __precisionZeros); } + + if ((exp & 0x00000100) != 0) { value = mulDivFloor(value, __base8, __precisionZeros); } + if ((exp & 0x00000200) != 0) { value = mulDivFloor(value, __base9, __precisionZeros); } + if ((exp & 0x00000400) != 0) { value = mulDivFloor(value, __base10, __precisionZeros); } + if ((exp & 0x00000800) != 0) { value = mulDivFloor(value, __base11, __precisionZeros); } + + if ((exp & 0x00001000) != 0) { value = mulDivFloor(value, __base12, __precisionZeros); } + if ((exp & 0x00002000) != 0) { value = mulDivFloor(value, __base13, __precisionZeros); } + if ((exp & 0x00004000) != 0) { value = mulDivFloor(value, __base14, __precisionZeros); } + if ((exp & 0x00008000) != 0) { value = mulDivFloor(value, __base15, __precisionZeros); } + + if ((exp & 0x00010000) != 0) { value = mulDivFloor(value, __base16, __precisionZeros); } + if ((exp & 0x00020000) != 0) { value = mulDivFloor(value, __base17, __precisionZeros); } + if ((exp & 0x00040000) != 0) { value = mulDivFloor(value, __base18, __precisionZeros); } + if ((exp & 0x00080000) != 0) { value = mulDivFloor(value, __base19, __precisionZeros); } + + if ((exp & 0x00100000) != 0) { value = mulDivFloor(value, __base20, __precisionZeros); } + if ((exp & 0x00200000) != 0) { value = mulDivFloor(value, __base21, __precisionZeros); } + if ((exp & 0x00400000) != 0) { value = mulDivFloor(value, __base22, __precisionZeros); } + if ((exp & 0x00800000) != 0) { value = mulDivFloor(value, __base23, __precisionZeros); } + + if ((exp & 0x01000000) != 0) { value = mulDivFloor(value, __base24, __precisionZeros); } + if ((exp & 0x02000000) != 0) { value = mulDivFloor(value, __base25, __precisionZeros); } + if ((exp & 0x04000000) != 0) { value = mulDivFloor(value, __base26, __precisionZeros); } + if ((exp & 0x08000000) != 0) { value = mulDivFloor(value, __base27, __precisionZeros); } + + if ((exp & 0x10000000) != 0) { value = mulDivFloor(value, __base28, __precisionZeros); } + if ((exp & 0x20000000) != 0) { value = mulDivFloor(value, __base29, __precisionZeros); } + if ((exp & 0x40000000) != 0) { value = mulDivFloor(value, __base30, __precisionZeros); } + + return value; +} + +@method_id(102) +fun test2(g1: bool) { + if (g1) { + return __base30; + } + return __base30; +} + +fun main() { + return 0x7fffffff; +} + +/** +@testcase | 101 | 12345678901234567890 0 | 12345678901234567890 +@testcase | 101 | 12345678901234567890 2147483647 | 1152091301147391687695287037940761486473052236056853112522947 +@testcase | 102 | 0 | 305482242253527345283975278648594775678 +@testcase | 102 | -1 | 305482242253527345283975278648594775678 + +@fif_codegen +""" + test2() PROC:<{ + IFJMP:<{ + 305482242253527345283975278648594775678 PUSHINT // '62 + }> + 305482242253527345283975278648594775678 PUSHINT // '124 + }> +""" + */ diff --git a/tolk-tester/tests/special-fun-names.tolk b/tolk-tester/tests/special-fun-names.tolk index 11c34f98d..8771762ab 100644 --- a/tolk-tester/tests/special-fun-names.tolk +++ b/tolk-tester/tests/special-fun-names.tolk @@ -22,8 +22,6 @@ fun test101() { } /** -@experimental_options remove-unused-functions - @testcase | 0 | | 0 @testcase | -1 | | -1 @testcase | -2 | | -2 diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 8eba3b427..eb33de0e1 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -569,6 +569,47 @@ fun test41(rev: bool) { } } +struct PrivPoint { + private readonly x: int + private y: int + z: int +} + +fun PrivPoint.create(x: int, y: int): PrivPoint { + return { x, y, z: 0 } +} + +fun PrivPoint.modifyY(mutate self, newY: int) { + self.y = newY +} + +fun PrivPoint.getSum(self) { + return self.x + self.y + self.z +} + +@method_id(142) +fun test42() { + var p1 = PrivPoint.create(10, 20); + p1.modifyY(80); + p1.z += 5; + return (p1, p1.getSum()); +} + +struct PrivWithDefaults { + private a: int32 = 0 + private b: bool = false +} + +fun PrivWithDefaults.createA(a: U) { + return PrivWithDefaults { a: a } +} + +@method_id(143) +fun test43() { + var obj: PrivWithDefaults = {}; + return (obj, PrivWithDefaults.createA(20)); +} + fun main(x: int8, y: MInt) { __expect_type(PointAlias{x,y}, "Point"); __expect_type(Point{x,y} as Point, "Point"); @@ -586,37 +627,37 @@ type PointAlias = Point; @testcase | 105 | | 35 30 45 @testcase | 106 | | 10 20 @testcase | 107 | | 5 5 10 20 15 -@testcase | 108 | | 777 typeid-3 777 0 777 777 777 typeid-3 777 typeid-4 777 777 +@testcase | 108 | | 777 typeid-2 777 0 777 777 777 typeid-2 777 typeid-3 777 777 @testcase | 109 | | 70 30 20 20 -80 @testcase | 110 | 0 | (null) (null) 0 -@testcase | 110 | -1 | 10 20 typeid-5 +@testcase | 110 | -1 | 10 20 typeid-1 @testcase | 111 | 0 | 0 2 10 20 30 @testcase | 111 | -1 | 0 0 3 4 7 @testcase | 112 | 0 | (null) (null) 0 -@testcase | 112 | -1 | 1 2 typeid-5 +@testcase | 112 | -1 | 1 2 typeid-1 @testcase | 113 | | (null) (null) 0 @testcase | 114 | | 1 2 @testcase | 115 | | (null) (null) (null) (null) 0 -@testcase | 116 | | -1 -4 [ 8 4 ] 7 (null) 0 [ 10 11 ] typeid-7 +@testcase | 116 | | -1 -4 [ 8 4 ] 7 (null) 0 [ 10 11 ] typeid-5 @testcase | 117 | 5 | 5 5 5 5 5 @testcase | 117 | null | (null) (null) (null) -1 (null) @testcase | 118 | | 17 -@testcase | 119 | | 10 20 9 10 typeid-5 9 -@testcase | 120 | 0 | 80 9 8 80 typeid-5 +@testcase | 119 | | 10 20 9 10 typeid-1 9 +@testcase | 120 | 0 | 80 9 8 80 typeid-1 @testcase | 120 | -1 | 8 14 (null) (null) 0 @testcase | 121 | 0 | 17 17 8 136 @testcase | 121 | -1 | 8 (null) 8 8 @testcase | 122 | 100 | 101 50 106 212 100 101 101 @testcase | 124 | 66 | 66 -@testcase | 125 | | (null) (null) 40 60 typeid-5 +@testcase | 125 | | (null) (null) 40 60 typeid-1 @testcase | 126 | | 118 -@testcase | 127 | | 0 0 typeid-3 typeid-3 777 -1 -1 0 0 777 0 0 typeid-10 777 0 0 777 (null) (null) 0 -@testcase | 128 | -1 | (null) (null) typeid-3 -1 0 0 777 (null) (null) typeid-3 -1 0 0 -@testcase | 128 | 0 | 1 2 typeid-2 0 -1 0 777 0 typeid-3 typeid-10 0 -1 0 +@testcase | 127 | | 0 0 typeid-2 typeid-2 777 -1 -1 0 0 777 0 0 typeid-7 777 0 0 777 (null) (null) 0 +@testcase | 128 | -1 | (null) (null) typeid-2 -1 0 0 777 (null) (null) typeid-2 -1 0 0 +@testcase | 128 | 0 | 1 2 typeid-8 0 -1 0 777 0 typeid-2 typeid-7 0 -1 0 @testcase | 129 | | (null) -@testcase | 130 | -1 | typeid-12 777 (null) typeid-12 777 0 777 0 0 -@testcase | 130 | 0 | typeid-11 777 4 1 777 typeid-11 777 0 0 -@testcase | 131 | | 5 typeid-13 (null) typeid-13 (null) 0 777 0 0 -1 777 0 -1 +@testcase | 130 | -1 | typeid-9 777 (null) typeid-9 777 0 777 0 0 +@testcase | 130 | 0 | typeid-10 777 4 1 777 typeid-10 777 0 0 +@testcase | 131 | | 5 typeid-11 (null) typeid-11 (null) 0 777 0 0 -1 777 0 -1 @testcase | 132 | | 10 20 10 0 0 20 0 0 0 0 @testcase | 133 | | -1 0 5 (null) (null) (null) 0 0 46 @testcase | 134 | | 10 20 @@ -628,6 +669,8 @@ type PointAlias = Point; @testcase | 140 | | 2 1 4 3 @testcase | 141 | -1 | 4 3 2 1 @testcase | 141 | 0 | 3 4 1 2 +@testcase | 142 | | 10 80 5 95 +@testcase | 143 | | 0 0 20 0 @fif_codegen """ diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index 136c606d2..23ca55c46 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -175,7 +175,7 @@ fun alwaysThrowX(x: int): never { } @noinline -fun anotherNever(throw123: bool): never { +fun anotherNever(throw123: bool) { // auto-infer `never` if (throw123) { alwaysThrow123(); } alwaysThrowX(456); } @@ -264,6 +264,8 @@ fun testBigExcno() { } fun main() { + // mark used to codegen them + testCodegen3; } /** diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index 3bc0e5114..587e16433 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -8,6 +8,8 @@ type Pair2_v1 = | (int, int) type Pair2_v2 = (MInt, MInt) type MBool = | bool type Tuple2Int = [int, int] +type UserId = int +type OwnerId = int struct Point { x: int; y: int } type PointAlias = Point; @@ -35,9 +37,15 @@ fun test1(x: MInt): MVoid { __expect_type(rand() ? (1, 2) : (1, 2) as Pair2_v1, "(int, int)"); __expect_type(rand() ? (1, 2) : (1, 2) as Pair2_v2, "(int, int)"); __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2), "Pair2_v1"); - __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "(int, int)"); __expect_type(rand() ? (1, 2) as Pair2_v2 : (1, 2) as Pair2_v2, "Pair2_v2"); + // since `Pair2_v1` and `Pair2_v2` underlying type is equal, a union of them can't be created + var d1 = (1, 2) as Pair2_v1 | Pair2_v2; + var d2 = (1, 2) as Pair2_v2 | Pair2_v1; + __expect_type(d1, "Pair2_v1"); + __expect_type(d2, "Pair2_v2"); + __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "Pair2_v1"); + __expect_type(!x, "bool"); __expect_type(x as int?, "int?"); @@ -63,9 +71,9 @@ fun test3(x: MIntN, y: MInt?) { if (x != null) { __expect_type(x, "MInt"); } - __expect_type(x, "MInt?"); + __expect_type(x, "MIntN"); var (z1, z2) = (x, x!, ); - __expect_type(z1, "MInt?"); + __expect_type(z1, "MIntN"); __expect_type(z2, "MInt"); } @@ -76,9 +84,9 @@ fun test4(x: MIntN, y: MIntN) { __expect_type(x + y, "int"); return x + y; } - __expect_type(x, "MInt?"); + __expect_type(x, "MIntN"); __expect_type(x!, "MInt"); - __expect_type(rand() ? x : y, "MInt?"); + __expect_type(rand() ? x : y, "MIntN"); __expect_type(rand() ? x : 0, "MInt?"); __expect_type(rand() ? 0 : x, "int?"); __expect_type(rand() ? 0 : x!, "int"); @@ -97,8 +105,8 @@ fun test5() { __expect_type(x, "Pair2_v1"); __expect_type(y, "Pair2_v2"); - takeTensor_v1(x); takeTensor_v2(x); takeTensor_v3(x); - takeTensor_v1(y); takeTensor_v2(y); takeTensor_v3(y); + takeTensor_v1(x); takeTensor_v2(x as Pair2_v2); takeTensor_v3(x); + takeTensor_v1(y as Pair2_v1); takeTensor_v2(y); takeTensor_v3(y); takeTensor_v1(z); takeTensor_v2(z); takeTensor_v3(z); var t = (y, x); @@ -117,16 +125,16 @@ fun test6() { __expect_type(z.0, "MInt"); } -fun someFn1(v: MInt): MIntN { return v; } +fun someFn1(v: MInt): MIntN { return v as MIntN; } fun test7() { var f1: (int) -> int? = someFn1; var f2: (int) -> MInt? = someFn1; - var f3: (MInt_v2) -> MInt_v2? = someFn1; + var f3: (MInt_v2) -> MInt_v2? = someFn1 as (MInt_v2) -> MInt_v2?; f1 = f2; f1 = f3; - f2 = f1; f2 = f3; - f3 = f1; f3 = f2; + f2 = f1; f2 = f3 as (int) -> MInt?; + f3 = f1; f3 = f2 as (MInt_v2) -> MInt_v2?; } fun test8() { @@ -143,6 +151,7 @@ fun test8() { someFn1 as (MInt_v2) -> MInt_v2?; } +@method_id(109) fun test9(b: MBool): MBool { if (!b) { __expect_type(b, "MBool"); @@ -156,7 +165,7 @@ fun test10() { var x1: Pair2_v1 = (5, 6); var (a1, b1) = x1; __expect_type(a1, "int"); - var x2: Pair2_v2? = x1; + var x2: Pair2_v2? = x1 as (int, int); var (a2, b2) = x2; __expect_type(a2, "MInt"); var x3: Tuple2Int = [9, 10]; @@ -198,6 +207,188 @@ fun test12(makeNotNull: bool) { return t; } +struct Wrapper { + value: T; +} +type WrapperOfInt = Wrapper +type WrapperOfMInt = Wrapper +type WrapperAlias = Wrapper +type WrapperAliasOfInt = WrapperAlias +type WrapperAliasOfMInt = WrapperAlias + +fun WrapperOfInt.myFn(self): int1 { return 0 } +fun WrapperOfMInt.myFn(self): int2 { return 0 } +fun WrapperAliasOfInt.myFn(self): int3 { return 0 } +fun WrapperAliasOfMInt.myFn(self): int4 { return 0 } + +fun test13(a: WrapperAlias, b: WrapperAlias, c: Wrapper, d: Wrapper, + e: WrapperAliasOfInt, f: WrapperAliasOfMInt, g: WrapperOfInt, h: WrapperOfMInt) { + __expect_type(g.myFn(), "int1"); + __expect_type(h.myFn(), "int2"); + __expect_type(e.myFn(), "int3"); + __expect_type(f.myFn(), "int4"); + + a = a; a = b; a = c; a = d; a = e; + b = a; b = b; b = c; b = d; b = f; + c = a; c = b; c = c; c = d; c = e; c = f; c = g; c = h; + d = a; d = b; d = c; d = d; d = e; d = f; d = g; d = h; + e = a; e = c; e = d; e = e; + f = b; f = c; f = d; f = f; + g = c; g = d; g = g; + h = c; h = d; h = h; + + a as WrapperAlias; + a as WrapperAlias; + a as Wrapper; + a as Wrapper; + a as WrapperOfInt; + a as WrapperOfMInt; + a as WrapperAliasOfInt; + a as WrapperAliasOfMInt; + + match (a) { WrapperAlias => {} } + match (a) { WrapperAlias => {} } + match (a) { Wrapper => {} } + match (a) { Wrapper => {} } + // match (a) { WrapperOfInt => {} } + // match (a) { WrapperOfMInt => {} } + match (a) { WrapperAliasOfInt => {} } + // match (a) { WrapperAliasOfMInt => {} } + + b as WrapperAlias; + b as WrapperAlias; + b as Wrapper; + b as Wrapper; + b as WrapperOfInt; + b as WrapperOfMInt; + b as WrapperAliasOfInt; + b as WrapperAliasOfMInt; + + match (b) { WrapperAlias => {} } + match (b) { WrapperAlias => {} } + match (b) { Wrapper => {} } + match (b) { Wrapper => {} } + // match (b) { WrapperOfInt => {} } + // match (b) { WrapperOfMInt => {} } + // match (b) { WrapperAliasOfInt => {} } + match (b) { WrapperAliasOfMInt => {} } + + match (e) { WrapperAliasOfInt => {} } + + match (f) { WrapperAliasOfMInt => {} } + match (f) { Wrapper => {} } + match (f) { Wrapper => {} } + // match (f) { WrapperAlias => {} } + match (f) { WrapperAlias => {} } + + __expect_type(a, "WrapperAlias"); + __expect_type(b, "WrapperAlias"); + __expect_type(c, "Wrapper"); + __expect_type(d, "Wrapper"); + __expect_type(e, "WrapperAliasOfInt"); + __expect_type(f, "WrapperAliasOfMInt"); + __expect_type(g, "WrapperOfInt"); + __expect_type(h, "WrapperOfMInt"); + + __expect_type(g.myFn(), "int1"); + __expect_type(h.myFn(), "int2"); + __expect_type(e.myFn(), "int3"); + __expect_type(f.myFn(), "int4"); +} + +type Payload14 = MInt | MBool | PointAlias2 + +fun test14(i: MInt, b: MBool, c: PointAlias2) { + var p: Payload14 = i; + p = b; + p = c; + p = i as int; + p = b as bool; + p = c as Point; + p = c as PointAlias; +} + +type MInt1 = int +type MInt2 = MInt1 + +fun int.plus0(self) { return self + 0 } +fun MInt1.plus1(self) { return self + 1 } +fun MInt2.plus2(self) { return self + 2 } + +@method_id(115) +fun test15() { + var i: int = 0; + var m: MInt1 = 10; + var t: MInt2 = 20; + + if (0) { + i = m; i = t; + m = i; m = t; + t = i; t = m; + } + + return ( + i.plus0(), i.plus1(), i.plus2(), + m.plus0(), m.plus1(), m.plus2(), + t.plus0(), t.plus1(), t.plus2(), + ) +} + +type StrangeUnionInt = int | UserId +type StrangeUnionInt2 = UserId | OwnerId + +struct ManyUnionFields { + f1: int | UserId + f2: StrangeUnionInt + f3: UserId | OwnerId + f4: OwnerId | UserId + f5: int | StrangeUnionInt2 + f6: StrangeUnionInt2 + + f10: slice | int | UserId + f11: slice | UserId | int + f12: OwnerId | () | int | UserId | slice + f13: OwnerId | cell | StrangeUnionInt2? +} + +fun test16(a: ManyUnionFields) { + __expect_type(a.f1, "int"); + __expect_type(a.f2, "StrangeUnionInt"); + __expect_type(a.f3, "UserId"); + __expect_type(a.f5, "int"); + __expect_type(a.f6, "StrangeUnionInt2"); + a.f1 + a.f2 + a.f3 + a.f4 + a.f5 + a.f6; + + __expect_type(a.f10, "slice | int"); + __expect_type(a.f11, "slice | UserId"); + __expect_type(a.f12, "OwnerId | () | slice"); + __expect_type(a.f13, "OwnerId | cell | null"); + + match (a.f13) { + int => {} + cell => {} + null => {} + } +} + +type SnakedCell = cell + +fun test17(a: SnakedCell, b: SnakedCell, c: SnakedCell) { + a = a; a = b; a = c; + b = a; + c = a; + + a as SnakedCell; + a as SnakedCell; + a as SnakedCell; + b as SnakedCell; + b as SnakedCell; + b as SnakedCell; + c as SnakedCell; + c as SnakedCell; + c as SnakedCell; +} + fun main(x: MInt, y: MInt?) { return y == null ? x : x + y; @@ -209,6 +400,7 @@ fun main(x: MInt, y: MInt?) { @testcase | 110 | | 41 @testcase | 112 | 0 | [ 1 1 ] @testcase | 112 | -1 | [ 2 1 ] +@testcase | 115 | | 0 1 2 10 11 12 20 21 22 @fif_codegen """ diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 7b9d52b61..0f649f75e 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -222,7 +222,7 @@ fun test21() { __expect_type(a, "never"); } if (a is slice || a is int) { - __expect_type(a, "slice | int") + __expect_type(a, "int | slice") } else { __expect_type(a, "never") } @@ -254,12 +254,12 @@ fun test23(p2: bool) { if (p is (int, int) && 10 < 0) { __expect_type(p, "(int, int)"); } - __expect_type(p, "(int, int) | (int, int, int)"); + __expect_type(p, "Pair2Or3"); if (p is int) { __expect_type(p, "never"); return (0, 0); } - __expect_type(p, "(int, int) | (int, int, int)"); + __expect_type(p, "Pair2Or3"); if (p !is (int, int)) { __expect_type(p, "(int, int, int)"); p = (10, 20); @@ -552,7 +552,7 @@ fun test48(a: IntOrInt8OrInt16) { var b = a; __expect_type(b, "IntOrInt8OrInt16"); match (b) { MInt => {} int8 => {} int16 => {} } - __expect_type(b, "int | int8 | int16"); + __expect_type(b, "IntOrInt8OrInt16"); match (b) { int => {}, int8 => { return ((0, null), (0, null)); }, int16 => "" }; __expect_type(b, "int | int16"); return (checkGeneric3(a), checkGeneric3(5 as int | slice | builder)); @@ -579,10 +579,10 @@ fun test50() { test49(match (a) { int => a, slice => a, null => a, }); __expect_type(b, "VeryComplexType"); match (b) { int => {} Pair2 => {} builder => {} null => {} } - __expect_type(b, "int | (int, int) | builder | null"); - __expect_type(b!, "int | (int, int) | builder"); + __expect_type(b, "VeryComplexType"); + __expect_type(b!, "int | (MInt, int) | builder"); match (b) { int => 100, Pair2 => {} builder => return -2, null => "null" } - __expect_type(b, "int | (int, int) | null"); + __expect_type(b, "int | Pair2 | null"); if (b !is null) { match (b) { int => {}, Pair2 => {} } } else { @@ -796,7 +796,7 @@ fun main() { @testcase | 107 | 3 | 6 7 8 typeid-2 6 7 8 typeid-2 @testcase | 108 | | (null) 2 3 typeid-1 (null) (null) (null) 0 (null) 2 3 typeid-1 (null) 2 3 typeid-1 (null) (null) (null) 0 @testcase | 109 | | 6 7 8 typeid-2 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 -@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 typeid-7 +@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 typeid-3 @testcase | 120 | | 5 @testcase | 121 | | -1 0 0 -1 -1 0 @testcase | 122 | 0 0 1 | 36 @@ -810,19 +810,19 @@ fun main() { @testcase | 127 | | 1 2 5 1 @testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 typeid-1 @testcase | 129 | | (null) (null) 0 (null) 5 1 -@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-8 777 (null) (null) 0 -@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-8 777 (null) (null) 0 +@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-4 777 (null) (null) 0 +@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-4 777 (null) (null) 0 @testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 typeid-1 @testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 typeid-1 777 (null) (null) (null) 0 @testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 typeid-1 777 (null) (null) (null) 0 @testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 typeid-1 777 1 2 typeid-1 777 (null) 5 42 777 5 42 777 5 42 @testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 typeid-1 777 (null) (null) 1 2 typeid-1 777 (null) (null) 0 -@testcase | 136 | | 1 2 typeid-1 777 1 2 typeid-12 777 1 2 typeid-13 777 (null) 1 2 typeid-14 +@testcase | 136 | | 1 2 typeid-1 777 1 2 typeid-5 777 1 2 typeid-6 777 (null) 1 2 typeid-7 @testcase | 137 | 1 2 | 1 2 777 1 2 777 1 2 777 [ 1 2 ] -@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 typeid-18 777 1 (null) 0 2 typeid-19 777 1 5 1 2 typeid-19 777 1 (null) 5 1 2 typeid-20 +@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 typeid-9 777 1 (null) 0 2 typeid-10 777 1 5 1 2 typeid-10 777 1 (null) 5 1 2 typeid-11 @testcase | 139 | | 1 (null) (null) (null) (null) 0 2 777 1 (null) 0 2 777 1 5 1 2 777 1 (null) 5 1 2 @testcase | 140 | 5 | 5 42 -@testcase | 140 | 15 | 15 typeid-3 +@testcase | 140 | 15 | 15 typeid-12 @testcase | 140 | 90 | 90 49 @testcase | 141 | 7 8 | 7 7 1 8 8 1 @testcase | 141 | null null | (null) (null) 0 (null) (null) 0 @@ -843,9 +843,9 @@ fun main() { @testcase | 154 | | 100 1 @testcase | 155 | | 5 1 5 1 @testcase | 156 | 1 2 -1 | 2 44 2 2 44 2 2 44 2 44 2 2 2 -@testcase | 157 | 1 | (null) 0 (null) typeid-21 777 1 2 typeid-1 777 typeid-21 typeid-21 777 0 0 -1 0 -1 0 0 777 -1 0 -@testcase | 157 | 0 | (null) 0 (null) typeid-21 777 (null) (null) typeid-21 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 -@testcase | 158 | | (null) (null) typeid-21 (null) (null) typeid-21 (null) 0 (null) +@testcase | 157 | 1 | (null) 0 (null) typeid-14 777 1 2 typeid-1 777 typeid-14 typeid-14 777 0 0 -1 0 -1 0 0 777 -1 0 +@testcase | 157 | 0 | (null) 0 (null) typeid-14 777 (null) (null) typeid-14 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 +@testcase | 158 | | (null) (null) typeid-14 (null) (null) typeid-14 (null) 0 (null) @testcase | 159 | 0 4 | 456 @testcase | 159 | null 0 | 123 @testcase | 160 | | 10 0 diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index 0c59c6715..d5ecd4070 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -23,6 +23,7 @@ fun demo1(v: int): int { global demo_var: int; const demo_10: int = 10; +@method_id(101) fun test1(): int { var demo_var: int = demo_10; var demo_slice: int = demo_20; @@ -33,6 +34,7 @@ fun test1(): int { return demo_var + demo_slice; } +@method_id(102) fun test2() { return second; } diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 702c9fddc..f40e1718e 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -101,7 +101,7 @@ fun testVarApplyInTernary() { return t; } -fun always_throw2(x: int) { +fun always_throw2(x: int): void { throw 239 + x } @@ -309,7 +309,7 @@ fun testMethodsOfGenericStruct() { __expect_type(w2, "Wrapper"); var eq = Wrapper.equalsNotNull; __expect_type(eq, "(Wrapper, Wrapper) -> bool"); - var creator3 = Wrapper.createFromNull; + var creator3 = Wrapper.createFromNull; __expect_type(creator3, "() -> Wrapper"); return (eq(w1, w2), w2.value = w1.value as int, eq(w1, w2), creator3()); } @@ -350,7 +350,7 @@ fun main() {} @testcase | 107 | | 65537 @testcase | 108 | | 4 @testcase | 109 | | 0 3 0 7 -@testcase | 110 | 5 | 5 10 100 100 100 typeid-6 +@testcase | 110 | 5 | 5 10 100 100 100 typeid-2 @testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 @testcase | 111 | 2 3 9 | -1 @testcase | 111 | 11 22 44 | -1 @@ -360,7 +360,7 @@ fun main() {} @testcase | 112 | -1 -10 -20 | -1 @testcase | 113 | 0 | 12 12 @testcase | 113 | -1 | 12 -1 -@testcase | 114 | -1 | 2 3 typeid-3 +@testcase | 114 | -1 | 2 3 typeid-1 @testcase | 114 | 0 | (null) 12 1 @testcase | 115 | | 7 6 7 @testcase | 116 | | 7 11 80 diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js index dcd667e6e..fa17bd5fb 100644 --- a/tolk-tester/tolk-tester.js +++ b/tolk-tester/tolk-tester.js @@ -279,7 +279,7 @@ class TolkTestFile { /** @type {boolean} */ this.enable_tolk_lines_comments = false /** @type {number} */ - this.pivot_typeid = 138 // may be changed when stdlib introduces new union types + this.pivot_typeid = 128 } parse_input_from_tolk_file() { @@ -500,29 +500,31 @@ function copyFromCString(mod, ptr) { function compileFile(mod, filename, experimentalOptions, withSrcLineComments) { // see tolk-wasm.cpp: typedef void (*WasmFsReadCallback)(int, char const*, char**, char**) const callbackPtr = mod.addFunction((kind, dataPtr, destContents, destError) => { - if (kind === 0) { // realpath - try { - let relative = copyFromCString(mod, dataPtr) - if (relative.startsWith('@stdlib/')) { - // import "@stdlib/filename" or import "@stdlib/filename.tolk" - relative = STDLIB_FOLDER + '/' + relative.substring(7) - if (!relative.endsWith('.tolk')) { - relative += '.tolk' + switch (kind) { // enum ReadCallback::Kind in C++ + case 0: // realpath + let relativeFilename = copyFromCString(mod, dataPtr) // from `import` statement, relative to cur file + if (!relativeFilename.endsWith('.tolk')) { + relativeFilename += '.tolk' + } + copyToCStringPtr(mod, path.normalize(relativeFilename), destContents) + break + case 1: // read file + try { + const filename = copyFromCString(mod, dataPtr) // already normalized (as returned above) + if (filename.startsWith('@stdlib/')) { + const contents = fs.readFileSync(STDLIB_FOLDER + '/' + filename.substring(8)).toString('utf-8'); + copyToCStringPtr(mod, contents, destContents) + } else { + const contents = fs.readFileSync(filename).toString('utf-8'); + copyToCStringPtr(mod, contents, destContents) } + } catch (err) { + copyToCStringPtr(mod, err.message || err.toString(), destError) } - copyToCStringPtr(mod, fs.realpathSync(relative), destContents); - } catch (err) { - copyToCStringPtr(mod, 'cannot find file', destError); - } - } else if (kind === 1) { // read file - try { - const absolute = copyFromCString(mod, dataPtr) // already normalized (as returned above) - copyToCStringPtr(mod, fs.readFileSync(absolute).toString('utf-8'), destContents); - } catch (err) { - copyToCStringPtr(mod, err.message || err.toString(), destError); - } - } else { - copyToCStringPtr(mod, 'Unknown callback kind=' + kind, destError); + break + default: + copyToCStringPtr(mod, 'Unknown callback kind=' + kind, destError) + break } }, 'viiii'); diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index da81f87fe..4fba75448 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -266,7 +266,7 @@ def __init__(self, tolk_filename: str, artifacts_folder: str): self.expected_hash: TolkTestCaseExpectedHash | None = None self.experimental_options: str | None = None self.enable_tolk_lines_comments = False - self.pivot_typeid = 138 # may be changed when stdlib introduces new union types + self.pivot_typeid = 128 def parse_input_from_tolk_file(self): with open(self.tolk_filename, "r") as fd: diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 033456e7f..6657be95d 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -10,6 +10,7 @@ set(TOLK_SOURCE pack-unpack-api.cpp pack-unpack-serializers.cpp send-message-api.cpp + maps-kv-api.cpp pipe-discover-parse-sources.cpp pipe-register-symbols.cpp pipe-resolve-identifiers.cpp @@ -19,6 +20,7 @@ set(TOLK_SOURCE pipe-check-inferred-types.cpp pipe-refine-lvalue-for-mutate.cpp pipe-check-rvalue-lvalue.cpp + pipe-check-private-fields.cpp pipe-check-pure-impure.cpp pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp @@ -33,6 +35,7 @@ set(TOLK_SOURCE smart-casts-cfg.cpp generics-helpers.cpp lazy-helpers.cpp + overload-resolution.cpp abscode.cpp analyzer.cpp asmops.cpp diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index 8e9fe914e..5d70b8064 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -24,7 +24,7 @@ namespace tolk { // for instance, variables after their call aren't considered used // its main purpose is `throw` statement, it's a call to a built-in `__throw` function static bool does_function_always_throw(FunctionPtr fun_ref) { - return fun_ref->declared_return_type == TypeDataNever::create(); + return fun_ref->inferred_return_type == TypeDataNever::create(); } /* @@ -718,7 +718,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } - std::get(f_sym->body)->compile(tmp, res, args, loc); + std::get(f_sym->body)->compile(tmp, res, args, loc); if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 39cef55b7..03016a5fe 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -425,6 +425,11 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector = builtin + lex.next(); + return createV(loc); + } + AnyTypeV underlying_type = parse_type_from_tokens(lex); for (auto v_annotation : annotations) { @@ -946,23 +951,24 @@ static AnyExprV parse_expr75(Lexer& lex) { return parse_expr80(lex); } -// parse E as / is / !is +// parse E as / is / !is (left-to-right) static AnyExprV parse_expr40(Lexer& lex) { AnyExprV lhs = parse_expr75(lex); - if (lex.tok() == tok_as) { - SrcLocation loc = lex.cur_location(); - lex.next(); - AnyTypeV cast_to_type = parse_type_from_tokens(lex); - lhs = createV(loc, lhs, cast_to_type); - } else if (lex.tok() == tok_is) { + TokenType t = lex.tok(); + while (t == tok_as || t == tok_is) { SrcLocation loc = lex.cur_location(); lex.next(); AnyTypeV rhs_type = parse_type_from_tokens(lex); - bool is_negated = lhs->kind == ast_not_null_operator; // `a !is ...`, now lhs = `a!` - if (is_negated) { - lhs = lhs->as()->get_expr(); + if (t == tok_as) { + lhs = createV(loc, lhs, rhs_type); + } else { + bool is_negated = lhs->kind == ast_not_null_operator; // `a !is ...`, now lhs = `a!` + if (is_negated) { + lhs = lhs->as()->get_expr(); + } + lhs = createV(loc, lhs, rhs_type, is_negated); } - lhs = createV(loc, lhs, rhs_type, is_negated); + t = lex.tok(); } return lhs; } @@ -1122,7 +1128,7 @@ static V parse_block_statement(Lexer& lex) { static AnyV parse_return_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_return, "`return`"); - AnyExprV child = lex.tok() == tok_semicolon // `return;` actually means "nothing" (inferred as void) + AnyExprV child = lex.tok() == tok_semicolon || lex.tok() == tok_clbrace ? createV(lex.cur_location()) : parse_expr(lex); return createV(loc, child); @@ -1536,6 +1542,19 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); @@ -1548,7 +1567,7 @@ static AnyV parse_struct_field(Lexer& lex) { default_value = parse_expr(lex); } - return createV(loc, v_ident, default_value, declared_type); + return createV(loc, v_ident, is_private, is_readonly, default_value, declared_type); } static V parse_struct_body(Lexer& lex) { @@ -1620,6 +1639,64 @@ static AnyV parse_struct_declaration(Lexer& lex, const std::vector(loc, v_ident, genericsT_list, overflow1023_policy, opcode, parse_struct_body(lex)); } +static AnyV parse_enum_member(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.check(tok_identifier, "member name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + + AnyExprV init_value = nullptr; + if (lex.tok() == tok_assign) { // `Red = 1` + lex.next(); + init_value = parse_expr(lex); + } + + return createV(loc, v_ident, init_value); +} + +static V parse_enum_body(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_opbrace, "`{`"); + + std::vector members; + while (lex.tok() != tok_clbrace) { + members.push_back(parse_enum_member(lex)); + if (lex.tok() == tok_comma || lex.tok() == tok_semicolon) { + lex.next(); + } + } + lex.expect(tok_clbrace, "`}`"); + + return createV(loc, std::move(members)); +} + +static AnyV parse_enum_declaration(Lexer& lex, const std::vector>& annotations) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_enum, "`enum`"); + + lex.check(tok_identifier, "identifier"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + + AnyTypeV colon_type = nullptr; + if (lex.tok() == tok_colon) { // enum Role: int8 + lex.next(); + colon_type = parse_type_expression(lex); + } + + for (auto v_annotation : annotations) { + switch (v_annotation->kind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + default: + v_annotation->error("this annotation is not applicable to enum"); + } + } + + return createV(loc, v_ident, colon_type, parse_enum_body(lex)); +} + static AnyV parse_tolk_required_version(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.next_special(tok_semver, "semver"); // syntax: "tolk 0.6" @@ -1702,9 +1779,12 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { toplevel_declarations.push_back(parse_struct_declaration(lex, annotations)); annotations.clear(); break; + case tok_enum: + toplevel_declarations.push_back(parse_enum_declaration(lex, annotations)); + annotations.clear(); + break; case tok_export: - case tok_enum: case tok_operator: case tok_infix: lex.error("`" + static_cast(lex.cur_str()) +"` is not supported yet"); diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 581babee3..6b4a08f05 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -210,9 +210,11 @@ class ASTReplacerInFunctionBody : public ASTReplacer { }; +const std::vector& get_all_builtin_functions(); const std::vector& get_all_not_builtin_functions(); const std::vector& get_all_declared_constants(); const std::vector& get_all_declared_structs(); +const std::vector& get_all_declared_enums(); template void replace_ast_of_all_functions() { diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 19956e467..1f41dc3d1 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -235,7 +235,7 @@ class ASTReplicator final { return createV(v->loc, clone(v->get_params())); } static V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), v->default_value ? clone(v->default_value) : nullptr, clone(v->type_node)); + return createV(v->loc, clone(v->get_identifier()), v->is_private, v->is_readonly, v->default_value ? clone(v->default_value) : nullptr, clone(v->type_node)); } static V clone(V v) { return createV(v->loc, clone(v->get_all_fields())); diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index fd19401e9..570ab048c 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -102,6 +102,9 @@ class ASTStringifier final : public ASTVisitor { {ast_struct_field, "ast_struct_field"}, {ast_struct_body, "ast_struct_body"}, {ast_struct_declaration, "ast_struct_declaration"}, + {ast_enum_member, "ast_enum_member"}, + {ast_enum_body, "ast_enum_body"}, + {ast_enum_declaration, "ast_enum_declaration"}, {ast_tolk_required_version, "ast_tolk_required_version"}, {ast_import_directive, "ast_import_directive"}, {ast_tolk_file, "ast_tolk_file"}, @@ -176,6 +179,8 @@ class ASTStringifier final : public ASTVisitor { return static_cast(v->as()->get_identifier()->name) + ": " + ast_type_node_to_string(v->as()->type_node); case ast_struct_declaration: return "struct " + static_cast(v->as()->get_identifier()->name); + case ast_enum_declaration: + return "enum " + static_cast(v->as()->get_identifier()->name); case ast_assign: return "="; case ast_set_assign: @@ -361,6 +366,9 @@ class ASTStringifier final : public ASTVisitor { case ast_struct_field: return handle_vertex(v->as()); case ast_struct_body: return handle_vertex(v->as()); case ast_struct_declaration: return handle_vertex(v->as()); + case ast_enum_member: return handle_vertex(v->as()); + case ast_enum_body: return handle_vertex(v->as()); + case ast_enum_declaration: return handle_vertex(v->as()); case ast_tolk_required_version: return handle_vertex(v->as()); case ast_import_directive: return handle_vertex(v->as()); case ast_tolk_file: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 6e38ba792..dc18496b8 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -211,9 +211,12 @@ class ASTVisitorFunctionBody : public ASTVisitor { }; +const std::vector& get_all_builtin_functions(); const std::vector& get_all_not_builtin_functions(); +const std::vector& get_all_declared_global_vars(); const std::vector& get_all_declared_constants(); const std::vector& get_all_declared_structs(); +const std::vector& get_all_declared_enums(); template void visit_ast_of_all_functions() { diff --git a/tolk/ast.cpp b/tolk/ast.cpp index b22f9d31f..b13d90beb 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -177,6 +177,10 @@ void Vertex::assign_alias_ref(AliasDefPtr alias_ref) this->alias_ref = alias_ref; } +void Vertex::assign_enum_ref(EnumDefPtr enum_ref) { + this->enum_ref = enum_ref; +} + void Vertex::assign_struct_ref(StructPtr struct_ref) { this->struct_ref = struct_ref; } diff --git a/tolk/ast.h b/tolk/ast.h index 8836d2a06..1b748a58b 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -132,6 +132,9 @@ enum ASTNodeKind { ast_struct_field, ast_struct_body, ast_struct_declaration, + ast_enum_member, + ast_enum_body, + ast_enum_declaration, ast_tolk_required_version, ast_import_directive, ast_tolk_file, @@ -716,12 +719,14 @@ struct Vertex final : ASTExprUnary { typedef std::variant< FunctionPtr, // for `t.tupleAt` target is `tupleAt` global function StructFieldPtr, // for `user.id` target is field `id` of struct `User` + EnumMemberPtr, // for `Color.Red` target is `Red` enum member int // for `t.0` target is "indexed access" 0 > DotTarget; DotTarget target = static_cast(nullptr); // filled at type inferring bool is_target_fun_ref() const { return std::holds_alternative(target); } bool is_target_struct_field() const { return std::holds_alternative(target); } + bool is_target_enum_member() const { return std::holds_alternative(target); } bool is_target_indexed_access() const { return std::holds_alternative(target); } AnyExprV get_obj() const { return child; } @@ -1348,14 +1353,16 @@ template<> // ast_struct_field is one field at struct declaration // example: `struct Point { x: int, y: int }` is struct declaration, its body contains 2 fields struct Vertex final : ASTOtherVararg { + bool is_private; // declared as `private field: int` + bool is_readonly; // declared as `readonly field: int` AnyTypeV type_node; // always exists, typing struct fields is mandatory AnyExprV default_value; // nullptr if no default auto get_identifier() const { return children.at(0)->as(); } - Vertex(SrcLocation loc, V name_identifier, AnyExprV default_value, AnyTypeV type_node) + Vertex(SrcLocation loc, V name_identifier, bool is_private, bool is_readonly, AnyExprV default_value, AnyTypeV type_node) : ASTOtherVararg(ast_struct_field, loc, {name_identifier}) - , type_node(type_node), default_value(default_value) {} + , is_private(is_private), is_readonly(is_readonly), type_node(type_node), default_value(default_value) {} }; template<> @@ -1366,8 +1373,8 @@ struct Vertex final : ASTOtherVararg { auto get_field(int i) const { return children.at(i)->as(); } const std::vector& get_all_fields() const { return children; } - Vertex(SrcLocation loc, std::vector&& fields) - : ASTOtherVararg(ast_struct_body, loc, std::move(fields)) {} + Vertex(SrcLocation loc, std::vector&& members) + : ASTOtherVararg(ast_struct_body, loc, std::move(members)) {} }; template<> @@ -1393,6 +1400,50 @@ struct Vertex final : ASTOtherVararg { , genericsT_list(genericsT_list), overflow1023_policy(overflow1023_policy) {} }; +template<> +// ast_enum_member is one member at enum declaration +// example: `enum Color { Red = 1, Green, Blue }` is enum declaration, its body contains 3 members +struct Vertex final : ASTOtherVararg { + AnyExprV init_value; // nullptr if no default + + auto get_identifier() const { return children.at(0)->as(); } + + Vertex(SrcLocation loc, V name_identifier, AnyExprV init_value) + : ASTOtherVararg(ast_enum_member, loc, {name_identifier}) + , init_value(init_value) {} +}; + +template<> +// ast_enum_body is `{ ... }` inside enum declaration, it contains enum members +// example: `enum Color { Red = 1, Green, Blue }` its body contains 3 members +struct Vertex final : ASTOtherVararg { + int get_num_members() const { return size(); } + auto get_member(int i) const { return children.at(i)->as(); } + const std::vector& get_all_members() const { return children; } + + Vertex(SrcLocation loc, std::vector&& members) + : ASTOtherVararg(ast_enum_body, loc, std::move(members)) {} +}; + +template<> +// ast_enum_declaration is a declaring a `enum` similar to TypeScript (not to Rust, we don't need structural enums, we have unions) +// example: `enum Color { Red, Green, Blue }` +// example: `enum Role: int8 { Admin, User }` +struct Vertex final : ASTOtherVararg { + EnumDefPtr enum_ref = nullptr; // filled after register + AnyTypeV colon_type = nullptr; // serialization type after `:` if exists + + auto get_identifier() const { return children.at(0)->as(); } + auto get_enum_body() const { return children.at(1)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_enum_ref(EnumDefPtr enum_ref); + + Vertex(SrcLocation loc, V name_identifier, AnyTypeV colon_type, V enum_body) + : ASTOtherVararg(ast_enum_declaration, loc, {name_identifier, enum_body}) + , colon_type(colon_type) {} +}; + template<> // ast_tolk_required_version is a preamble fixating compiler's version at the top of the file // example: `tolk 0.6` diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index dc2795c89..fd48cc033 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -40,22 +40,38 @@ static std::vector define_builtin_parameters(const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags) { - auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); +static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const std::function& func, int flags) { + auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltinAsmOp(func), nullptr); G.symtable.add_function(f_sym); + G.all_builtins.push_back(f_sym); } -static void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, +static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const std::function& func, int flags) { + auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltinGenerateOps(func), nullptr); + G.symtable.add_function(f_sym); + G.all_builtins.push_back(f_sym); +} + +static void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const std::function& func, int flags, std::initializer_list arg_order = {}, std::initializer_list ret_order = {}) { std::string method_name = name.substr(name.find('.') + 1); - auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltinAsmOp(func), nullptr); f_sym->arg_order = arg_order; f_sym->ret_order = ret_order; G.symtable.add_function(f_sym); + G.all_builtins.push_back(f_sym); G.all_methods.push_back(f_sym); } -void FunctionBodyBuiltin::compile(AsmOpList& dest, std::vector& out, std::vector& in, +void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const std::function& func, int flags) { + std::string method_name = name.substr(name.find('.') + 1); + auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltinGenerateOps(func), nullptr); + G.symtable.add_function(f_sym); + G.all_builtins.push_back(f_sym); + G.all_methods.push_back(f_sym); +} + +void FunctionBodyBuiltinAsmOp::compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation loc) const { dest << simple_compile(out, in, loc); } @@ -1081,12 +1097,10 @@ static AsmOp compile_store_int(std::vector& res, std::vector // purpose: to merge consecutive `b.storeUint(0, 1).storeUint(1, 1)` into one "1 PUSHINT + 2 STU", // when constant arguments are passed, keep them as a separate (fake) instruction, to be handled by optimizer later bool value_and_len_is_const = z.is_int_const() && x.is_int_const(); - if (value_and_len_is_const && G.settings.optimization_level >= 2) { + if (value_and_len_is_const && x.int_const >= 0 && z.int_const > 0 && z.int_const <= 256 && G.settings.optimization_level >= 2) { // don't handle negative numbers or potential overflow, merging them is incorrect - bool value_is_safe = sgnd - ? x.int_const >= 0 && z.int_const < 64 && x.int_const < (1ULL << (z.int_const->to_long() - 1)) - : x.int_const >= 0; - if (value_is_safe && z.int_const > 0 && z.int_const <= (255 + !sgnd)) { + int len = static_cast(z.int_const->to_long()); + if (x.int_const->fits_bits(len, sgnd)) { z.unused(); x.unused(); return AsmOp::Custom(loc, "MY_store_int"s + (sgnd ? "I " : "U ") + x.int_const->to_dec_string() + " " + z.int_const->to_dec_string(), 1); @@ -1166,7 +1180,7 @@ static AsmOp compile_fetch_slice(std::vector& res, std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_slice_sdbeginsq(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 3 && res.size() == 2); auto& prefix = args[1]; auto& prefix_len = args[2]; @@ -1181,7 +1195,7 @@ AsmOp compile_slice_sdbeginsq(std::vector& res, std::vector& } // fun slice.skipBits(mutate self, len: int): self "SDSKIPFIRST" -AsmOp compile_skip_bits_in_slice(std::vector& res, std::vector& args, SrcLocation loc) { +static AsmOp compile_skip_bits_in_slice(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); auto& len = args[1]; // same technique as for storeUint: @@ -1268,7 +1282,7 @@ static AsmOp compile_any_object_sizeof(std::vector& res, std::vector&, std::vector&, SrcLocation loc) { +static AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation loc) { // all ton() invocations are constants, replaced by integers; no dynamic values allowed, no work at runtime tolk_assert(false); return AsmOp::Nop(loc); @@ -1292,6 +1306,64 @@ static AsmOp compile_expect_type(std::vector&, std::vector&, return AsmOp::Nop(loc); } +// implemented in dedicated files + +using GenerateOpsImpl = FunctionBodyBuiltinGenerateOps::GenerateOpsImpl; +using CompileToAsmOpImpl = FunctionBodyBuiltinAsmOp::CompileToAsmOpImpl; + +GenerateOpsImpl generate_T_toCell; +GenerateOpsImpl generate_builder_storeAny; +GenerateOpsImpl generate_T_fromSlice; +GenerateOpsImpl generate_slice_loadAny; +GenerateOpsImpl generate_T_fromCell; +GenerateOpsImpl generate_T_forceLoadLazyObject; +GenerateOpsImpl generate_slice_skipAny; +GenerateOpsImpl generate_T_estimatePackSize; + +GenerateOpsImpl generate_createMessage; +GenerateOpsImpl generate_createExternalLogMessage; +GenerateOpsImpl generate_address_buildInAnotherShard; +GenerateOpsImpl generate_AutoDeployAddress_buildAddress; +GenerateOpsImpl generate_AutoDeployAddress_addressMatches; + +GenerateOpsImpl generate_mapKV_exists; +GenerateOpsImpl generate_mapKV_get; +GenerateOpsImpl generate_mapKV_mustGet; +GenerateOpsImpl generate_mapKV_set; +GenerateOpsImpl generate_mapKV_setGet; +GenerateOpsImpl generate_mapKV_replace; +GenerateOpsImpl generate_mapKV_replaceGet; +GenerateOpsImpl generate_mapKV_add; +GenerateOpsImpl generate_mapKV_addGet; +GenerateOpsImpl generate_mapKV_del; +GenerateOpsImpl generate_mapKV_delGet; +GenerateOpsImpl generate_mapKV_findFirst; +GenerateOpsImpl generate_mapKV_findLast; +GenerateOpsImpl generate_mapKV_findKeyGreater; +GenerateOpsImpl generate_mapKV_findKeyGreaterOrEqual; +GenerateOpsImpl generate_mapKV_findKeyLess; +GenerateOpsImpl generate_mapKV_findKeyLessOrEqual; +GenerateOpsImpl generate_mapKV_iterateNext; +GenerateOpsImpl generate_mapKV_iteratePrev; + +CompileToAsmOpImpl compile_createEmptyMap; +CompileToAsmOpImpl compile_createMapFromLowLevelDict; +CompileToAsmOpImpl compile_dict_get; +CompileToAsmOpImpl compile_dict_mustGet; +CompileToAsmOpImpl compile_dict_getMin; +CompileToAsmOpImpl compile_dict_getMax; +CompileToAsmOpImpl compile_dict_getNext; +CompileToAsmOpImpl compile_dict_getNextEq; +CompileToAsmOpImpl compile_dict_getPrev; +CompileToAsmOpImpl compile_dict_getPrevEq; +CompileToAsmOpImpl compile_dict_set; +CompileToAsmOpImpl compile_dict_setGet; +CompileToAsmOpImpl compile_dict_replace; +CompileToAsmOpImpl compile_dict_replaceGet; +CompileToAsmOpImpl compile_dict_add; +CompileToAsmOpImpl compile_dict_addGet; +CompileToAsmOpImpl compile_dict_del; +CompileToAsmOpImpl compile_dict_delGet; void define_builtins() { using namespace std::placeholders; @@ -1432,7 +1504,7 @@ void define_builtins() { define_builtin_func("__throw", ParamsInt1, Never, nullptr, compile_throw, 0); - define_builtin_func("__throw_arg", {typeT, Int}, Never, declGenericT, + define_builtin_func("__throw_arg", {TypeDataUnknown::create(), Int}, Never, nullptr, compile_throw_arg, 0); define_builtin_func("__throw_if_unless", ParamsInt3, Unit, nullptr, @@ -1546,8 +1618,8 @@ void define_builtins() { compile_tuple_set_at, FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); define_builtin_method("address.buildSameAddressInAnotherShard", Address, {Address, AddressShardingOptions}, Builder, nullptr, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); + generate_address_buildInAnotherShard, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_method("debug.print", debug, {typeT}, Unit, declGenericT, compile_debug_print_to_string, FunctionData::flagAllowAnyWidthT); @@ -1564,17 +1636,17 @@ void define_builtins() { // serialization/deserialization methods to/from cells (or, more low-level, slices/builders) // they work with structs (or, more low-level, with arbitrary types) define_builtin_method("T.toCell", typeT, {typeT, PackOptions}, CellT, declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + generate_T_toCell, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); define_builtin_method("T.fromCell", typeT, {TypeDataCell::create(), UnpackOptions}, typeT, declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + generate_T_fromCell, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); define_builtin_method("T.fromSlice", typeT, {Slice, UnpackOptions}, typeT, declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); - define_builtin_method("T.estimatePackSize", typeT, {}, TypeDataBrackets::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + generate_T_fromSlice, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.estimatePackSize", typeT, {}, TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), declReceiverT, + generate_T_estimatePackSize, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); define_builtin_method("T.getDeclaredPackPrefix", typeT, {}, Int, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); @@ -1582,33 +1654,33 @@ void define_builtins() { compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); define_builtin_method("T.forceLoadLazyObject", typeT, {typeT}, Slice, declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + generate_T_forceLoadLazyObject, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); define_builtin_method("Cell.load", CellT, {CellT, UnpackOptions}, typeT, declReceiverT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + generate_T_fromCell, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); define_builtin_method("slice.loadAny", Slice, {Slice, UnpackOptions}, typeT, declGenericT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + generate_slice_loadAny, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); define_builtin_method("slice.skipAny", Slice, {Slice, UnpackOptions}, Slice, declGenericT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + generate_slice_skipAny, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); define_builtin_method("builder.storeAny", Builder, {Builder, typeT, PackOptions}, Builder, declGenericT, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + generate_builder_storeAny, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); define_builtin_func("createMessage", {CreateMessageOptions}, OutMessage, declTBody, - compile_time_only_function, - FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + generate_createMessage, + FunctionData::flagAllowAnyWidthT); define_builtin_func("createExternalLogMessage", {CreateExternalLogMessageOptions}, OutMessage, declTBody, - compile_time_only_function, - FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + generate_createExternalLogMessage, + FunctionData::flagAllowAnyWidthT); define_builtin_method("AutoDeployAddress.buildAddress", AutoDeployAddress, {AutoDeployAddress}, Builder, nullptr, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); + generate_AutoDeployAddress_buildAddress, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_method("AutoDeployAddress.addressMatches", AutoDeployAddress, {AutoDeployAddress, Address}, Bool, nullptr, - compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); + generate_AutoDeployAddress_addressMatches, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); // functions not presented in stdlib at all // used in tolk-tester to check/expose internal compiler state @@ -1625,6 +1697,121 @@ void define_builtins() { define_builtin_method("T.__toTuple", typeT, {typeT}, TypeDataTuple::create(), declReceiverT, compile_any_object_to_tuple, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + + TypePtr MapKV = TypeDataMapKV::create(TypeDataGenericT::create("K"), TypeDataGenericT::create("V")); + TypePtr TKey = TypeDataGenericT::create("K"); + TypePtr TValue = TypeDataGenericT::create("V"); + TypePtr LookupResultT = TypeDataUnknown::create(); + TypePtr EntryKV = TypeDataUnknown::create(); + const GenericsDeclaration* declGenericMapKV = new GenericsDeclaration(std::vector{{"K", nullptr}, {"V", nullptr}}, 0); + const GenericsDeclaration* declReceiverMapKV = new GenericsDeclaration(std::vector{{"K", nullptr}, {"V", nullptr}}, 2); + + // high-level methods for maps; + // they are generic, so all type checks are done automatically; + // but all calls to them are handled at generating Ops from AST, their "simple compile" is not called + define_builtin_func("createEmptyMap", {}, MapKV, declGenericMapKV, + compile_createEmptyMap, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); + define_builtin_func("createMapFromLowLevelDict", {TypeDataUnion::create_nullable(TypeDataCell::create())}, MapKV, declGenericMapKV, + compile_createMapFromLowLevelDict, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.exists", MapKV, {MapKV, TKey}, TypeDataBool::create(), declReceiverMapKV, + generate_mapKV_exists, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.get", MapKV, {MapKV, TKey}, LookupResultT, declReceiverMapKV, + generate_mapKV_get, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.mustGet", MapKV, {MapKV, TKey, TypeDataInt::create()}, TValue, declReceiverMapKV, + generate_mapKV_mustGet, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.set", MapKV, {MapKV, TKey, TValue}, TypeDataVoid::create(), declReceiverMapKV, + generate_mapKV_set, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT | FunctionData::flagReturnsSelf); + define_builtin_method("map.setAndGetPrevious", MapKV, {MapKV, TKey, TValue}, LookupResultT, declReceiverMapKV, + generate_mapKV_setGet, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.replaceIfExists", MapKV, {MapKV, TKey, TValue}, TypeDataBool::create(), declReceiverMapKV, + generate_mapKV_replace, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.replaceAndGetPrevious", MapKV, {MapKV, TKey, TValue}, LookupResultT, declReceiverMapKV, + generate_mapKV_replaceGet, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.addIfNotExists", MapKV, {MapKV, TKey, TValue}, TypeDataBool::create(), declReceiverMapKV, + generate_mapKV_add, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.addOrGetExisting", MapKV, {MapKV, TKey, TValue}, LookupResultT, declReceiverMapKV, + generate_mapKV_addGet, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.delete", MapKV, {MapKV, TKey}, TypeDataBool::create(), declReceiverMapKV, + generate_mapKV_del, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.deleteAndGetDeleted", MapKV, {MapKV, TKey}, LookupResultT, declReceiverMapKV, + generate_mapKV_delGet, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findFirst", MapKV, {MapKV}, EntryKV, declReceiverMapKV, + generate_mapKV_findFirst, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findLast", MapKV, {MapKV}, EntryKV, declReceiverMapKV, + generate_mapKV_findLast, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findKeyGreater", MapKV, {MapKV, TKey}, EntryKV, declReceiverMapKV, + generate_mapKV_findKeyGreater, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findKeyGreaterOrEqual", MapKV, {MapKV, TKey}, EntryKV, declReceiverMapKV, + generate_mapKV_findKeyGreaterOrEqual, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findKeyLess", MapKV, {MapKV, TKey}, EntryKV, declReceiverMapKV, + generate_mapKV_findKeyLess, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.findKeyLessOrEqual", MapKV, {MapKV, TKey}, EntryKV, declReceiverMapKV, + generate_mapKV_findKeyLessOrEqual, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.iterateNext", MapKV, {MapKV, EntryKV}, EntryKV, declReceiverMapKV, + generate_mapKV_iterateNext, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("map.iteratePrev", MapKV, {MapKV, EntryKV}, EntryKV, declReceiverMapKV, + generate_mapKV_iteratePrev, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + + // low-level functions that actually emit TVM assembly, they work on a "dict" level + TypePtr PlainDict = TypeDataCell::create(); + TypePtr KeySliceOrInt = TypeDataUnknown::create(); + TypePtr ValueSlice = TypeDataSlice::create(); + TypePtr ValueFound = TypeDataInt::create(); + TypePtr LookupSliceFound = TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}); + + define_builtin_func("__dict.get", {KeySliceOrInt, PlainDict, TypeDataInt::create()}, LookupSliceFound, nullptr, + compile_dict_get, 0); + define_builtin_func("__dict.mustGet", {KeySliceOrInt, PlainDict, TypeDataInt::create()}, LookupSliceFound, nullptr, + compile_dict_mustGet, 0); + define_builtin_func("__dict.getMin", {PlainDict}, TypeDataTensor::create({ValueSlice, KeySliceOrInt, ValueFound}), nullptr, + compile_dict_getMin, 0); + define_builtin_func("__dict.getMax", {PlainDict}, TypeDataTensor::create({ValueSlice, KeySliceOrInt, ValueFound}), nullptr, + compile_dict_getMax, 0); + define_builtin_func("__dict.getNext", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_getNext, 0); + define_builtin_func("__dict.getNextEq", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_getNextEq, 0); + define_builtin_func("__dict.getPrev", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_getPrev, 0); + define_builtin_func("__dict.getPrevEq", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_getPrevEq, 0); + define_builtin_func("__dict.set", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, PlainDict, nullptr, + compile_dict_set, 0); + define_builtin_func("__dict.setGet", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, LookupSliceFound}), nullptr, + compile_dict_setGet, 0); + define_builtin_func("__dict.replace", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_replace, 0); + define_builtin_func("__dict.replaceGet", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, LookupSliceFound}), nullptr, + compile_dict_replaceGet, 0); + define_builtin_func("__dict.add", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_add, 0); + define_builtin_func("__dict.addGet", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, LookupSliceFound}), nullptr, + compile_dict_addGet, 0); + define_builtin_func("__dict.del", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, TypeDataBool::create()}), nullptr, + compile_dict_del, 0); + define_builtin_func("__dict.delGet", {KeySliceOrInt, TypeDataSlice::create(), PlainDict, TypeDataInt::create()}, TypeDataTensor::create({PlainDict, LookupSliceFound}), nullptr, + compile_dict_delGet, 0); } // there are some built-in functions that operate on types declared in stdlib (like Cell) @@ -1696,6 +1883,35 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("createMessage")->mutate()->declared_return_type = OutMessage; lookup_function("createExternalLogMessage")->mutate()->parameters[0].declared_type = CreateExternalLogMessageOptions; lookup_function("createExternalLogMessage")->mutate()->declared_return_type = OutMessage; + + if (!lookup_global_symbol("MapLookupResult")) return; + StructPtr struct_ref_LookupResultT = lookup_global_symbol("MapLookupResult")->try_as(); + StructPtr struct_ref_EntryKV = lookup_global_symbol("MapEntry")->try_as(); + TypePtr TKey = TypeDataGenericT::create("K"); + TypePtr TValue = TypeDataGenericT::create("V"); + TypePtr LookupResultT = TypeDataGenericTypeWithTs::create(struct_ref_LookupResultT, nullptr, {TValue}); + TypePtr EntryKV = TypeDataGenericTypeWithTs::create(struct_ref_EntryKV, nullptr, {TKey, TValue}); + + lookup_function("map.get")->mutate()->declared_return_type = LookupResultT; + lookup_function("map.setAndGetPrevious")->mutate()->declared_return_type = LookupResultT; + lookup_function("map.replaceAndGetPrevious")->mutate()->declared_return_type = LookupResultT; + lookup_function("map.addOrGetExisting")->mutate()->declared_return_type = LookupResultT; + lookup_function("map.deleteAndGetDeleted")->mutate()->declared_return_type = LookupResultT; + + auto v_def_throwCode = createV({}, td::make_refint(9), "9"); + v_def_throwCode->assign_inferred_type(TypeDataInt::create()); + lookup_function("map.mustGet")->mutate()->parameters[2].assign_default_value(v_def_throwCode); + + lookup_function("map.findFirst")->mutate()->declared_return_type = EntryKV; + lookup_function("map.findLast")->mutate()->declared_return_type = EntryKV; + lookup_function("map.findKeyGreater")->mutate()->declared_return_type = EntryKV; + lookup_function("map.findKeyGreaterOrEqual")->mutate()->declared_return_type = EntryKV; + lookup_function("map.findKeyLess")->mutate()->declared_return_type = EntryKV; + lookup_function("map.findKeyLessOrEqual")->mutate()->declared_return_type = EntryKV; + lookup_function("map.iterateNext")->mutate()->declared_return_type = EntryKV; + lookup_function("map.iterateNext")->parameters[1].mutate()->declared_type = EntryKV; + lookup_function("map.iteratePrev")->mutate()->declared_return_type = EntryKV; + lookup_function("map.iteratePrev")->parameters[1].mutate()->declared_type = EntryKV; } } // namespace tolk diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index f5ef89de3..8be368810 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -373,7 +373,7 @@ bool Op::generate_code_step(Stack& stack) { if (f_sym->is_asm_function()) { std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args0) } else { - std::get(f_sym->body)->compile(stack.o, res, args0, loc); // compile res := f (args0) + std::get(f_sym->body)->compile(stack.o, res, args0, loc); // compile res := f (args0) } } else { stack.o << AsmOp::Custom(loc, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); @@ -511,7 +511,7 @@ bool Op::generate_code_step(Stack& stack) { if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } - std::get(f_sym->body)->compile(stack.o, res, args, loc); // compile res := f (args) + std::get(f_sym->body)->compile(stack.o, res, args, loc); // compile res := f (args) if (arg_order_already_equals_asm()) { maybe_swap_builtin_args_to_compile(); } diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp index f5bda29e6..f7e391984 100644 --- a/tolk/compiler-state.cpp +++ b/tolk/compiler-state.cpp @@ -43,9 +43,9 @@ void PersistentHeapAllocator::clear() { void CompilerSettings::enable_experimental_option(std::string_view name) { ExperimentalOption* to_enable = nullptr; - if (name == remove_unused_functions.name) { - to_enable = &remove_unused_functions; - } + // if (name == some_option.name) { + // to_enable = &some_option; + // } if (to_enable == nullptr) { std::cerr << "unknown experimental option: " << name << std::endl; @@ -66,10 +66,18 @@ void CompilerSettings::parse_experimental_options_cmd_arg(const std::string& cmd } } +const std::vector& get_all_builtin_functions() { + return G.all_builtins; +} + const std::vector& get_all_not_builtin_functions() { return G.all_functions; } +const std::vector& get_all_declared_global_vars() { + return G.all_global_vars; +} + const std::vector& get_all_declared_constants() { return G.all_constants; } @@ -78,4 +86,8 @@ const std::vector& get_all_declared_structs() { return G.all_structs; } +const std::vector& get_all_declared_enums() { + return G.all_enums; +} + } // namespace tolk diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index 6685e092a..cf3a11ffa 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -60,7 +60,7 @@ struct CompilerSettings { FsReadCallback read_callback; - ExperimentalOption remove_unused_functions{"remove-unused-functions"}; + // ExperimentalOption some_option{"some-option"}; void enable_experimental_option(std::string_view name); void parse_experimental_options_cmd_arg(const std::string& cmd_arg); @@ -96,12 +96,14 @@ struct CompilerState { GlobalSymbolTable symtable; PersistentHeapAllocator persistent_mem; + std::vector all_builtins; // all built-in functions std::vector all_functions; // all user-defined (not built-in) global-scope functions, with generic instantiations std::vector all_methods; // all user-defined and built-in extension methods for arbitrary types (receivers) std::vector all_contract_getters; std::vector all_global_vars; std::vector all_constants; std::vector all_structs; + std::vector all_enums; AllRegisteredSrcFiles all_src_files; bool is_verbosity(int gt_eq) const { return settings.verbosity >= gt_eq; } diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 20c316fdb..5952b8c06 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -315,8 +315,13 @@ struct ConstantExpressionChecker { if (auto v_cast_to = v->try_as()) { return visit(v_cast_to->get_expr()); } - if (auto v_dot = v->try_as(); v_dot && (v_dot->is_target_indexed_access() || v_dot->is_target_fun_ref())) { - return visit(v_dot->get_obj()); + if (auto v_dot = v->try_as()) { + if (v_dot->is_target_indexed_access()) { // anotherConst.0 + return visit(v_dot->get_obj()); + } + if (v_dot->is_target_enum_member()) { // Color.Red + return; + } } v->error("not a constant expression"); } @@ -329,6 +334,79 @@ struct ConstantExpressionChecker { } }; +struct EnumMemberEvaluator { + static td::RefInt256 handle_unary_operator(V v) { + td::RefInt256 rhs = visit(v->get_rhs()); + if (!rhs->is_valid()) return rhs; + + switch (v->tok) { + case tok_minus: return -rhs; + case tok_bitwise_not: return ~rhs; + case tok_plus: return rhs; + default: + fire(nullptr, v->loc, "unsupported operator in `enum` member value"); + } + } + + static td::RefInt256 handle_binary_operator(V v) { + td::RefInt256 lhs = visit(v->get_lhs()); + td::RefInt256 rhs = visit(v->get_rhs()); + if (!lhs->is_valid()) return lhs; + if (!rhs->is_valid()) return rhs; + + switch (v->tok) { + case tok_plus: return lhs + rhs; + case tok_minus: return lhs - rhs; + case tok_mul: return lhs * rhs; + case tok_div: return lhs / rhs; + case tok_mod: return lhs % rhs; + case tok_bitwise_and: return lhs & rhs; + case tok_bitwise_or: return lhs | rhs; + case tok_bitwise_xor: return lhs ^ rhs; + case tok_lshift: return lhs << static_cast(rhs->to_long()); + case tok_rshift: return lhs >> static_cast(rhs->to_long()); + default: + fire(nullptr, v->loc, "unsupported operator in `enum` member value"); + } + } + + // `enum { Red = SIX }`, we met `SIX` + static td::RefInt256 handle_reference(V v) { + GlobalConstPtr const_ref = v->sym->try_as(); + if (!const_ref) { + v->error("symbol `" + static_cast(v->get_name()) + "` is not a constant"); + } + return visit(const_ref->init_value); + } + + static td::RefInt256 visit(AnyExprV v) { + if (auto v_int = v->try_as()) { + return v_int->intval; + } + if (auto v_un = v->try_as()) { + return handle_unary_operator(v_un); + } + if (auto v_bin = v->try_as()) { + return handle_binary_operator(v_bin); + } + if (auto v_ref = v->try_as()) { + return handle_reference(v_ref); + } + if (auto v_par = v->try_as()) { + return visit(v_par->get_expr()); + } + if (auto v_dot = v->try_as()) { + // `enum E { V = AnotherEnum.A }` not allowed (can be allowed in the future, + // but remember to construct an initialization tree, since A may be auto-computed, and prevent recursion) + if (v_dot->is_target_enum_member()) { + fire(nullptr, v->loc, "referencing other members in initializers is not allowed yet"); + } + } + fire(nullptr, v->loc, "unsupported operation in `enum` member value"); + } +}; + + void check_expression_is_constant(AnyExprV v_expr) { ConstantExpressionChecker::check_expression_expected_to_be_constant(v_expr); } @@ -346,5 +424,9 @@ CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call) { return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } +td::RefInt256 eval_enum_member_init_value(AnyExprV init_value) { + return EnumMemberEvaluator::visit(init_value); +} + } // namespace tolk diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index b7a9fc7fe..4f488951c 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -27,5 +27,6 @@ typedef std::variant CompileTimeFunctionResult; void check_expression_is_constant(AnyExprV v_expr); std::string eval_string_const_standalone(AnyExprV v_string); CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call); +td::RefInt256 eval_enum_member_init_value(AnyExprV init_value); } // namespace tolk diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index f46def4ac..8a5028edb 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -36,6 +36,8 @@ struct GlobalConstData; struct AliasDefData; struct StructFieldData; struct StructData; +struct EnumMemberData; +struct EnumDefData; using LocalVarPtr = const LocalVarData*; using FunctionPtr = const FunctionData*; @@ -44,6 +46,8 @@ using GlobalConstPtr = const GlobalConstData*; using AliasDefPtr = const AliasDefData*; using StructFieldPtr = const StructFieldData*; using StructPtr = const StructData*; +using EnumMemberPtr = const EnumMemberData*; +using EnumDefPtr = const EnumDefData*; class TypeData; using TypePtr = const TypeData*; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index ae51cf752..4f76bc86c 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -89,7 +89,6 @@ void GenericsSubstitutions::set_typeT(std::string_view nameT, TypePtr typeT) { for (int i = 0; i < size(); ++i) { if (genericTs->get_nameT(i) == nameT) { if (valuesTs[i] == nullptr) { - tolk_assert(!typeT->has_genericT_inside()); valuesTs[i] = typeT; } break; @@ -126,6 +125,12 @@ GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(StructPtr struct_ref) , deducedTs(struct_ref->genericTs) { } +GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(const GenericsDeclaration* genericTs) + : fun_ref(nullptr) + , struct_ref(nullptr) + , deducedTs(genericTs) { +} + // purpose: having `f(value: T)` and call `f(5)`, deduce T = int // generally, there may be many generic Ts for declaration, and many arguments // for every argument, `consider_next_condition()` is called @@ -146,8 +151,16 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T deducedTs.set_typeT(asT->nameT, arg_type); } else if (const auto* p_nullable = param_type->try_as(); p_nullable && p_nullable->or_null) { // `arg: T?` called as `f(nullableInt)` => T is int - if (const auto* a_nullable = arg_type->unwrap_alias()->try_as(); a_nullable && a_nullable->or_null) { - consider_next_condition(p_nullable->or_null, a_nullable->or_null); + // `arg: T?` called as `f(T1|T2|null)` => T is T1|T2 + if (const auto* a_nullable = arg_type->unwrap_alias()->try_as(); a_nullable && a_nullable->has_null()) { + std::vector rest_but_null; + rest_but_null.reserve(a_nullable->size() - 1); + for (TypePtr a_variant : a_nullable->variants) { + if (a_variant != TypeDataNullLiteral::create()) { + rest_but_null.push_back(a_variant); + } + } + consider_next_condition(p_nullable->or_null, TypeDataUnion::create(std::move(rest_but_null))); } // `arg: T?` called as `f(int)` => T is int else { @@ -184,7 +197,9 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T bool is_sub_correct = true; for (TypePtr p_variant : p_union->variants) { if (!p_variant->has_genericT_inside()) { - auto it = std::find(a_sub_p.begin(), a_sub_p.end(), p_variant); + auto it = std::find_if(a_sub_p.begin(), a_sub_p.end(), [p_variant](TypePtr a) { + return a->equal_to(p_variant); + }); if (it != a_sub_p.end()) { a_sub_p.erase(it); } else { @@ -210,12 +225,33 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T } } else if (const auto* p_instSt = param_type->try_as(); p_instSt && p_instSt->struct_ref) { // `arg: Wrapper` called as `f(wrappedInt)` => T is int - if (const auto* a_struct = arg_type->try_as(); a_struct && a_struct->struct_ref->is_instantiation_of_generic_struct() && a_struct->struct_ref->base_struct_ref == p_instSt->struct_ref) { + if (const auto* a_struct = arg_type->unwrap_alias()->try_as(); a_struct && a_struct->struct_ref->is_instantiation_of_generic_struct() && a_struct->struct_ref->base_struct_ref == p_instSt->struct_ref) { tolk_assert(p_instSt->size() == a_struct->struct_ref->substitutedTs->size()); for (int i = 0; i < p_instSt->size(); ++i) { consider_next_condition(p_instSt->type_arguments[i], a_struct->struct_ref->substitutedTs->typeT_at(i)); } } + // `arg: Wrapper` called as `f(Wrapper>)` => T is Wrapper + if (const auto* a_instSt = arg_type->try_as(); a_instSt && a_instSt->struct_ref == p_instSt->struct_ref) { + tolk_assert(p_instSt->size() == a_instSt->size()); + for (int i = 0; i < p_instSt->size(); ++i) { + consider_next_condition(p_instSt->type_arguments[i], a_instSt->type_arguments[i]); + } + } + // `arg: Wrapper?` called as `f(Wrapper)` => T is int + if (const auto* a_union = arg_type->unwrap_alias()->try_as()) { + TypePtr variant_matches = nullptr; + int n_matches = 0; + for (TypePtr a_variant : a_union->variants) { + if (const auto* a_struct = a_variant->unwrap_alias()->try_as(); a_struct && a_struct->struct_ref->is_instantiation_of_generic_struct() && a_struct->struct_ref->base_struct_ref == p_instSt->struct_ref) { + variant_matches = a_variant; + n_matches++; + } + } + if (n_matches == 1) { + consider_next_condition(param_type, variant_matches); + } + } } else if (const auto* p_instAl = param_type->try_as(); p_instAl && p_instAl->alias_ref) { // `arg: WrapperAlias` called as `f(wrappedInt)` => T is int if (const auto* a_alias = arg_type->try_as(); a_alias && a_alias->alias_ref->is_instantiation_of_generic_alias() && a_alias->alias_ref->base_alias_ref == p_instAl->alias_ref) { @@ -224,6 +260,26 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T consider_next_condition(p_instAl->type_arguments[i], a_alias->alias_ref->substitutedTs->typeT_at(i)); } } + } else if (const auto* p_map = param_type->try_as()) { + // `arg: map` called as `f(someMapInt32Slice)` => K = int32, V = slice + if (const auto* a_map = arg_type->unwrap_alias()->try_as()) { + consider_next_condition(p_map->TKey, a_map->TKey); + consider_next_condition(p_map->TValue, a_map->TValue); + } + // `arg: map?` called as `f(someMapInt32Slice)` => K = int32, V = slice + if (const auto* a_union = arg_type->unwrap_alias()->try_as()) { + TypePtr variant_matches = nullptr; + int n_matches = 0; + for (TypePtr a_variant : a_union->variants) { + if (a_variant->unwrap_alias()->try_as()) { + variant_matches = a_variant; + n_matches++; + } + } + if (n_matches == 1) { + consider_next_condition(param_type, variant_matches); + } + } } } @@ -442,86 +498,26 @@ AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutio return new_alias_ref; } -// find `builder.storeInt` for called_receiver = "builder" and called_name = "storeInt" -// most practical case, when a direct method for receiver exists; -// note, that having an alias `type WorkchainNum = int` and methods `WorkchainNum.isMasterchain()`, -// it's okay to call `-1.isMasterchain()`, because int equals to any alias; -// currently there is no chance to change this logic, say, `type AssetList = dict` to have separate methods, -// due to smart casts, types merge of control flow rejoin, etc., which immediately become `cell?` -FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name) { - FunctionPtr exact_found = nullptr; - - for (FunctionPtr method_ref : G.all_methods) { - if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { - if (method_ref->receiver_type->equal_to(called_receiver)) { - if (exact_found) { - return nullptr; - } - exact_found = method_ref; - } - } - } - - return exact_found; -} - -// find `int?.copy` / `T.copy` for called_receiver = "int" and called_name = "copy" -std::vector match_methods_for_call_including_generic(TypePtr called_receiver, std::string_view called_name) { - std::vector candidates; - - // step1: find all methods where a receiver equals to provided, e.g. `MInt.copy` - for (FunctionPtr method_ref : G.all_methods) { - if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { - if (method_ref->receiver_type->equal_to(called_receiver)) { - candidates.emplace_back(method_ref, GenericsSubstitutions(method_ref->genericTs)); - } - } - } - if (!candidates.empty()) { - return candidates; +// a function `tuple.push(self, v: T) asm "TPUSH"` can't be called with T=Point (2 stack slots); +// almost all asm/built-in generic functions expect one stack slot, but there are exceptions +bool is_allowed_asm_generic_function_with_non1_width_T(FunctionPtr fun_ref, int idxT) { + // if a built-in function is marked with a special flag + if (fun_ref->is_variadic_width_T_allowed()) { + return true; } - // step2: find all methods where a receiver can accept provided, e.g. `int8.copy` / `int?.copy` / `(int|slice).copy` - for (FunctionPtr method_ref : G.all_methods) { - if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { - if (method_ref->receiver_type->can_rhs_be_assigned(called_receiver)) { - candidates.emplace_back(method_ref, GenericsSubstitutions(method_ref->genericTs)); - } + // allow "Cell.hash", "map.isEmpty" and other methods that don't depend on internal structure + if (fun_ref->is_method() && idxT < fun_ref->genericTs->n_from_receiver) { + TypePtr receiver = fun_ref->receiver_type->unwrap_alias(); + if (const auto* r_withTs = receiver->try_as()) { + return r_withTs->struct_ref && r_withTs->struct_ref->name == "Cell"; } - } - if (!candidates.empty()) { - return candidates; - } - - // step 3: try to match generic receivers, e.g. `Container.copy` / `(T?|slice).copy` but NOT `T.copy` - for (FunctionPtr method_ref : G.all_methods) { - if (method_ref->method_name == called_name && method_ref->receiver_type->has_genericT_inside() && !method_ref->receiver_type->try_as()) { - try { - GenericSubstitutionsDeducing deducingTs(method_ref); - TypePtr replaced = deducingTs.auto_deduce_from_argument(method_ref->receiver_type, called_receiver); - if (!replaced->has_genericT_inside()) { - candidates.emplace_back(method_ref, deducingTs.flush()); - } - } catch (...) {} + if (receiver->try_as()) { + return true; } } - if (!candidates.empty()) { - return candidates; - } - // step 4: try to match `T.copy` - for (FunctionPtr method_ref : G.all_methods) { - if (method_ref->method_name == called_name && method_ref->receiver_type->try_as()) { - try { - GenericSubstitutionsDeducing deducingTs(method_ref); - TypePtr replaced = deducingTs.auto_deduce_from_argument(method_ref->receiver_type, called_receiver); - if (!replaced->has_genericT_inside()) { - candidates.emplace_back(method_ref, deducingTs.flush()); - } - } catch (...) {} - } - } - return candidates; + return false; } } // namespace tolk diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index c8f46ff27..4cec94cd2 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -97,6 +97,7 @@ class GenericSubstitutionsDeducing { public: explicit GenericSubstitutionsDeducing(FunctionPtr fun_ref); explicit GenericSubstitutionsDeducing(StructPtr struct_ref); + explicit GenericSubstitutionsDeducing(const GenericsDeclaration* genericTs); TypePtr replace_Ts_with_currently_deduced(TypePtr orig) const; TypePtr auto_deduce_from_argument(TypePtr param_type, TypePtr arg_type); @@ -110,13 +111,10 @@ class GenericSubstitutionsDeducing { } }; -typedef std::pair MethodCallCandidate; - FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs); StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions&& substitutedTs); AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutions&& substitutedTs); -FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name); -std::vector match_methods_for_call_including_generic(TypePtr called_receiver, std::string_view called_name); +bool is_allowed_asm_generic_function_with_non1_width_T(FunctionPtr fun_ref, int idxT); } // namespace tolk diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 44d166768..c349a91ae 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -380,8 +380,10 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { break; case 7: if (str == "builtin") return tok_builtin; + if (str == "private") return tok_private; break; case 8: + if (str == "readonly") return tok_readonly; if (str == "continue") return tok_continue; if (str == "operator") return tok_operator; break; diff --git a/tolk/lexer.h b/tolk/lexer.h index 63f0e6a30..b92b2a1c2 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -44,6 +44,8 @@ enum TokenType { tok_colon, tok_asm, tok_builtin, + tok_private, + tok_readonly, tok_int_const, tok_string_const, diff --git a/tolk/maps-kv-api.cpp b/tolk/maps-kv-api.cpp new file mode 100644 index 000000000..770450ae3 --- /dev/null +++ b/tolk/maps-kv-api.cpp @@ -0,0 +1,723 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "maps-kv-api.h" +#include "type-system.h" +#include "generics-helpers.h" +#include "pack-unpack-api.h" + +/* + * `map` is a high-level wrapper over TVM dictionaries. + * The compiler automatically constructs correct DICT asm instructions, takes care of packing slices, etc. + * + * In practice, K is most likely intN or address, V is any serializable value. + * If K is numeric, DICTI* or DICTU* instructions are used. + * If K is address or bitsN, DICT* instructions are used (internal address assumed). + * If K is complex, it's automatically packed into a slice, and DICT* instructions are used. + * + * On writing, DICTSETB instructions (providing a builder). Later, if a value is constant (so that a slice pushed + * into this builder is constant), it's replaced no DICTSET (not B) with a peephole. + * For writing `map>`, DICTSETREF instruction is used, but ONLY for writing (not for getting!). + * REF instructions are not used for getting to maintain the same external interface `MapLookupResult` and `MapEntry` + * with `loadValue()` etc. + */ + +namespace tolk { + +enum class DictKeyKind { + IntKey, // DICTI* + UIntKey, // DICTU* + SliceKey, // DICT* +}; + +enum class DictValueKind { + SliceValue, // DICTSET + BuilderValue, // DICTSETB + CellRefValue, // DICTSETREF +}; + + +// map should use DICTI* instructions +// note that `struct UserId { v: int32 }` also optimized, since it's just a signed int on a stack +static int is_TKey_TVM_int(TypePtr TKey) { + if (const TypeDataIntN* t_intN = TKey->try_as(); t_intN && !t_intN->is_variadic && !t_intN->is_unsigned) { + return t_intN->n_bits; + } + if (const TypeDataAlias* t_alias = TKey->try_as()) { + return is_TKey_TVM_int(t_alias->underlying_type); + } + if (const TypeDataStruct* t_struct = TKey->try_as(); t_struct && t_struct->struct_ref->get_num_fields() == 1) { + return is_TKey_TVM_int(t_struct->struct_ref->get_field(0)->declared_type); + } + if (TKey == TypeDataBool::create()) { // allow `bool` as a key with `DICTI` instructions + return true; + } + return 0; +} + +// map should use DICTU* instructions +static int is_TKey_TVM_uint(TypePtr TKey) { + if (const TypeDataIntN* t_intN = TKey->try_as(); t_intN && !t_intN->is_variadic && t_intN->is_unsigned) { + return t_intN->n_bits; + } + if (const TypeDataAlias* t_alias = TKey->try_as()) { + return is_TKey_TVM_uint(t_alias->underlying_type); + } + if (const TypeDataStruct* t_struct = TKey->try_as(); t_struct && t_struct->struct_ref->get_num_fields() == 1) { + return is_TKey_TVM_uint(t_struct->struct_ref->get_field(0)->declared_type); + } + return 0; +} + +// map should use DICT* instructions +// note that map is forbidden, since a raw slice doesn't define binary width +static int is_TKey_TVM_slice(TypePtr TKey) { + if (TKey == TypeDataAddress::create()) { + return 3 + 8 + 256; // "internal address" assumed + } + if (const TypeDataBitsN* t_bitsN = TKey->try_as()) { + return t_bitsN->is_bits ? t_bitsN->n_width : t_bitsN->n_width * 8; + } + if (const TypeDataAlias* t_alias = TKey->try_as()) { + return is_TKey_TVM_slice(t_alias->underlying_type); + } + if (const TypeDataStruct* t_struct = TKey->try_as(); t_struct && t_struct->struct_ref->get_num_fields() == 1) { + return is_TKey_TVM_slice(t_struct->struct_ref->get_field(0)->declared_type); + } + return 0; +} + +// we allow `map` and handle it separately, because we don't need to unpack it +static bool is_TValue_raw_slice(TypePtr TValue) { + return TValue->unwrap_alias() == TypeDataSlice::create(); +} + +// `map>` can emit SETREF instructions +static bool is_TValue_cell_or_CellT(TypePtr TValue) { + return TValue->unwrap_alias() == TypeDataCell::create() || is_type_cellT(TValue->unwrap_alias()); +} + +bool check_mapKV_TKey_is_valid(TypePtr TKey, std::string& because_msg) { + if (is_TKey_TVM_int(TKey) || is_TKey_TVM_uint(TKey) || is_TKey_TVM_slice(TKey)) { + return true; + } + + // okay, not a trivial key — it must be a serializable struct of a constant size + if (TKey->unwrap_alias() == TypeDataSlice::create()) { // a dedicated error message for `map` + because_msg = "because it does not specify keyLen for a dictionary\nhint: use `address` if a key is an internal address\nhint: use `bits128` and similar if a key represents fixed-width data"; + return false; + } + if (!check_struct_can_be_packed_or_unpacked(TKey, false, because_msg)) { + because_msg = "because it can not be serialized to slice\n" + because_msg; + return false; + } + + PackSize pack_size = estimate_serialization_size(TKey); + if (pack_size.min_bits != pack_size.max_bits) { + because_msg += "because its binary size is not constant: it's " + std::to_string(pack_size.min_bits) + ".." + std::to_string(pack_size.max_bits) + " bits"; + return false; + } + if (pack_size.min_bits > 1023) { + because_msg += "because its binary size is too large: " + std::to_string(pack_size.min_bits) + " bits"; + return false; + } + if (pack_size.max_refs) { + because_msg += "because it may contain a cell reference, not only data bits"; + return false; + } + return true; +} + +bool check_mapKV_TValue_is_valid(TypePtr TValue, std::string& because_msg) { + // we allow `slice` and `RemainingBitsAndRefs` as a value + if (is_TValue_raw_slice(TValue)) { + return true; + } + // or something that can be packed to/from slice + if (!check_struct_can_be_packed_or_unpacked(TValue, false, because_msg)) { + because_msg = "because it can not be serialized\n" + because_msg; + return false; + } + // note that `struct A { s: slice }` can not be used as a value (not serializable), + // although `slice` can, because in stdlib behavior for TValue=slice is overloaded (no deserialization) + + return true; +} + + +// an internal helper, having TKey and TValue, generate IR variables passed to __dict.* built-in functions +class DictKeyValue { + CodeBlob& code; + SrcLocation loc; + + DictKeyKind key_kind; + int key_len; + var_idx_t key_irv = -1; + + DictValueKind value_kind; + var_idx_t value_irv = -1; + +public: + + var_idx_t ir_key_kind() const { + return code.create_int(loc, static_cast(key_kind), "(key-kind)"); + } + + var_idx_t ir_value_kind() const { + return code.create_int(loc, static_cast(value_kind), "(value-kind)"); + } + + var_idx_t ir_key_len() const { + return code.create_int(loc, key_len, "(key-len)"); + } + + var_idx_t ir_key_val() const { + tolk_assert(key_irv != -1); + return key_irv; + } + + var_idx_t ir_value_val() const { + tolk_assert(value_irv != -1); + return value_irv; + } + + DictKeyValue(CodeBlob& code, SrcLocation loc, TypePtr TKey, const std::vector* exact_key, TypePtr TValue, const std::vector* exact_value, bool allow_REF_TValue = false) + : code(code) + , loc(loc) { + if (int i_bits = is_TKey_TVM_int(TKey)) { + key_kind = DictKeyKind::IntKey; + key_len = i_bits; + if (exact_key != nullptr) { + tolk_assert(exact_key->size() == 1); + key_irv = exact_key->at(0); + } + } else if (int u_bits = is_TKey_TVM_uint(TKey)) { + key_kind = DictKeyKind::UIntKey; + key_len = u_bits; + if (exact_key != nullptr) { + tolk_assert(exact_key->size() == 1); + key_irv = exact_key->at(0); + } + } else if (int s_bits = is_TKey_TVM_slice(TKey)) { + key_kind = DictKeyKind::SliceKey; + key_len = s_bits; + if (exact_key != nullptr) { + tolk_assert(exact_key->size() == 1); + key_irv = exact_key->at(0); + } + } else { + key_kind = DictKeyKind::SliceKey; + PackSize pack_size = EstimateContext().estimate_any(TKey); + tolk_assert(pack_size.max_refs == 0 && pack_size.min_bits == pack_size.max_bits); + key_len = pack_size.max_bits; + + if (exact_key != nullptr) { + std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), loc, "(map-keyB)"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + ctx.generate_pack_any(TKey, std::vector(*exact_key)); + std::vector ir_slice = code.create_tmp_var(TypeDataSlice::create(), loc, "(map-key)"); + // todo BTOS + code.emplace_back(loc, Op::_Call, ir_slice, ir_builder, lookup_function("builder.endCell")); + code.emplace_back(loc, Op::_Call, ir_slice, ir_slice, lookup_function("cell.beginParse")); + key_irv = ir_slice[0]; + } + } + + if (is_TValue_raw_slice(TValue)) { + value_kind = DictValueKind::SliceValue; + if (exact_value != nullptr) { + value_irv = exact_value->at(0); + } + } else if (allow_REF_TValue && is_TValue_cell_or_CellT(TValue)) { + // note that we use CellRefValue for writing only (not for reading, not for "set+get"): + // we don't emit REF for getters to match typing of MapLookupResult and MapEntry, + // so that `loadValue()` implemented in stdlib works universally for any V (particularly, Cell) + // (given `map.get`, DICTGET will be emitted, and loadValue() will load a ref correctly) + value_kind = DictValueKind::CellRefValue; + if (exact_value != nullptr) { + value_irv = exact_value->at(0); + } + } else { + value_kind = DictValueKind::BuilderValue; + + if (exact_value != nullptr) { + std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), loc, "(valueB)"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + ctx.generate_pack_any(TValue, std::vector(*exact_value)); + value_irv = ir_builder[0]; + } + } + } +}; + +// MapEntry is a built-in struct { rawValue: slice, key: K, isFound: bool } +// when used for numeric K, tvm instructions DICTI* and DICTU* return an integer key onto the stack +// when used for address/bitsN, tvm instructions DICT* return a slice key onto the stack +// so, in practice, we don't need any transformations from a TVM result, +// but when K is complex (like struct Point), TVM instructions return a slice, which is needed to be unpacked to K +GNU_ATTRIBUTE_NOINLINE +static std::vector construct_MapEntry_with_non_trivial_key(CodeBlob& code, SrcLocation loc, std::vector&& ir_entry, TypePtr TKey) { + tolk_assert(ir_entry.size() == 3); // slice value, slice key, isFound + + std::vector ir_key = code.create_tmp_var(TKey, loc, "(entry-key)"); + Op& if_found = code.emplace_back(loc, Op::_If, std::vector{ir_entry[2]}); + { + code.push_set_cur(if_found.block0); + UnpackContext ctx(code, loc, std::vector(ir_entry.begin() + 1, ir_entry.begin() + 2), create_default_UnpackOptions(code, loc)); + std::vector ir_unpacked_key = ctx.generate_unpack_any(TKey); + code.emplace_back(loc, Op::_Let, ir_key, std::move(ir_unpacked_key)); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_found.block1); + for (var_idx_t ith_null : ir_key) { + code.emplace_back(loc, Op::_Let, std::vector{ith_null}, std::vector{ir_entry[1]}); + } + code.close_pop_cur(loc); + } + + std::vector ir_result; + ir_result.reserve(2 + ir_key.size()); + ir_result.push_back(ir_entry[0]); // rawSlice + ir_result.insert(ir_result.end(), ir_key.begin(), ir_key.end()); + ir_result.push_back(ir_entry[2]); // isFound + return ir_result; +} + +static std::vector create_ir_MapEntry(CodeBlob& code, SrcLocation loc) { + return code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create(), TypeDataInt::create()}), loc, "(entry)"); +} + +// see a comment above construct_MapEntry_with_non_trivial_key() +static std::vector finalize_ir_MapEntry(CodeBlob& code, SrcLocation loc, std::vector&& ir_entry, TypePtr TKey) { + if (!is_TKey_TVM_int(TKey) && !is_TKey_TVM_uint(TKey) && !is_TKey_TVM_slice(TKey)) { + ir_entry = construct_MapEntry_with_non_trivial_key(code, loc, std::move(ir_entry), TKey); + } + return ir_entry; +} + + +// ---------------------------------- +// generating AsmOp and IR code +// + + +// "DICTSET" -> "DICTISETB" having key_kind and value_kind +static std::string choose_dict_op(std::string_view op_slice, VarDescr& var_with_key_kind, VarDescr& var_with_value_kind) { + std::string op(op_slice); + + DictKeyKind key_kind = static_cast(var_with_key_kind.int_const->to_long()); + var_with_key_kind.unused(); + + if (key_kind == DictKeyKind::UIntKey) op = "DICTU" + op.substr(4); + if (key_kind == DictKeyKind::IntKey) op = "DICTI" + op.substr(4); + + DictValueKind value_kind = static_cast(var_with_value_kind.int_const->to_long()); + var_with_value_kind.unused(); + + if (value_kind == DictValueKind::BuilderValue) op += "B"; + if (value_kind == DictValueKind::CellRefValue) op += "REF"; + + return op; +} + +// "empty map" is just NULL in TVM; it's extracted as a built-in to check for K/V correctness in advance +AsmOp compile_createEmptyMap(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1 && args.empty()); + return AsmOp::Custom(loc, "NEWDICT", 0, 1); +} + +// "convert dict to map" is just NOP; it's extracted as a built-in to allow non-1 width K/V +AsmOp compile_createMapFromLowLevelDict(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1 && args.size() == 1); + return AsmOp::Parse(loc, "NOP"); +} + +// DICTGET: k D n => (x −1) OR (0); + NULLSWAPIFNOT => (x -1) OR (null 0) +AsmOp compile_dict_get(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 2 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGET", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 3, 2); +} + +// DICTGET: k D n => (x −1) OR (0) +// but since it's MUST get, it's wrapped to assert(found), so assume x (value) will exist on a stack +AsmOp compile_dict_mustGet(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 2 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGET", args[0], args[1]); + return AsmOp::Custom(loc, op, 3, 2); +} + +// DICTMIN: D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getMin(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+2); + std::string op = choose_dict_op("DICTMIN", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 2, 3); +} + +// DICTMAX: D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getMax(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+2); + std::string op = choose_dict_op("DICTMAX", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 2, 3); +} + +// DICTGETNEXT: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getNext(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGETNEXT", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); +} + +// DICTGETNEXTEQ: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getNextEq(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGETNEXTEQ", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); +} + +// DICTGETPREV: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getPrev(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGETPREV", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); +} + +// DICTGETPREVEQ: k D n => (x k −1) OR (0); + NULLSWAPIFNOT2 => (x k -1) OR (null null 0) +AsmOp compile_dict_getPrevEq(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+3); + std::string op = choose_dict_op("DICTGETPREVEQ", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT2", 3, 3); +} + +// DICTSET: x k D n => D' +AsmOp compile_dict_set(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1 && args.size() == 2+4); + std::string op = choose_dict_op("DICTSET", args[0], args[1]); + return AsmOp::Custom(loc, op, 4, 1); +} + +// DICTSETGET: x k D n => (D' y −1) or (D' 0); + NULLSWAPIFNOT => (D' y -1) OR (D' null 0) +AsmOp compile_dict_setGet(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+4); + std::string op = choose_dict_op("DICTSETGET", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 4, 3); +} + +// DICTREPLACE: x k D n => (D' -1) OR (D 0) +AsmOp compile_dict_replace(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 2 && args.size() == 2+4); + std::string op = choose_dict_op("DICTREPLACE", args[0], args[1]); + return AsmOp::Custom(loc, op, 4, 2); +} + +// DICTREPLACEGET: x k D n => (D' y -1) OR (D 0); + NULLSWAPIFNOT => (D' y -1) OR (D null 0) +AsmOp compile_dict_replaceGet(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+4); + std::string op = choose_dict_op("DICTREPLACEGET", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 4, 3); +} + +// DICTADD: x k D n => (D' -1) OR (D 0) +AsmOp compile_dict_add(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 2 && args.size() == 2+4); + std::string op = choose_dict_op("DICTADD", args[0], args[1]); + return AsmOp::Custom(loc, op, 4, 2); +} + +// DICTADDGET: x k D n => (D' -1) OR (D y 0); + NULLSWAPIF + NOT => (D' null 0) OR (D y -1) (from "isAdded" to "isFound") +AsmOp compile_dict_addGet(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+4); + std::string op = choose_dict_op("DICTADDGET", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIF" + " NOT", 4, 3); +} + +// DICTDEL: k D n => (D' -1) OR (D 0) +AsmOp compile_dict_del(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 2 && args.size() == 2+3); + std::string op = choose_dict_op("DICTDEL", args[0], args[1]); + return AsmOp::Custom(loc, op, 3, 2); +} + +// DICTDELGET: k D n => (D' x -1) OR (D 0); + NULLSWAPIFNOT => (D' x -1) OR (D null 0) +AsmOp compile_dict_delGet(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 3 && args.size() == 2+3); + std::string op = choose_dict_op("DICTDELGET", args[0], args[1]); + return AsmOp::Custom(loc, op + " NULLSWAPIFNOT", 3, 3); +} + + +// fun map.exists(self, key: K): bool +std::vector generate_mapKV_exists(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.get")); + + return {ir_lookup[1]}; // isFound from (sliceOrNull isFound) +} + +// fun map.get(self, key: K): MapLookupResult +std::vector generate_mapKV_get(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.get")); + + // in all functions where we return MapLookupResult: + // on a stack we have (slice, found) - exactly the shape of MapLookupResult; + // the user manually calls `result.loadValue()` after checking result.isFound + return ir_lookup; +} + +// fun map.mustGet(self, key: K, throwIfNotFound: int = 9): V +std::vector generate_mapKV_mustGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + // since we don't return MapLookupResult, for `map>` we can use DICTGETREF + bool use_DICTGETREF = is_TValue_cell_or_CellT(TValue); + DictKeyValue kv(code, loc, TKey, &args[1], use_DICTGETREF ? TValue : TypeDataSlice::create(), nullptr, true); + + std::vector ir_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_lookup, std::move(dict_args), lookup_function("__dict.mustGet")); + + std::vector args_assert = { args[2][0], ir_lookup[1], code.create_int(loc, 0, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); + + std::vector ir_slice(ir_lookup.begin(), ir_lookup.begin() + 1); + if (is_TValue_raw_slice(TValue)) { + return ir_slice; + } + if (use_DICTGETREF) { // ir_slice holds a cell actually + return ir_slice; // (it's exactly what we need, we need to return Cell) + } + + // load TValue and check for assertEnd (it's the default behavior) + UnpackContext ctx(code, loc, std::move(ir_slice), create_default_UnpackOptions(code, loc)); + std::vector ir_value = ctx.generate_unpack_any(TValue); + ctx.assertEndIfOption(); + return ir_value; +} + +// fun map.set(mutate self, key: K, value: V): self +std::vector generate_mapKV_set(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2], true); + + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, args[0], std::move(dict_args), lookup_function("__dict.set")); + + return args[0]; // return mutated map +} + +// fun map.setAndGetPrevious(mutate self, key: K, value: V): MapLookupResult +std::vector generate_mapKV_setGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2]); + + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(map-and-lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.setGet")); + + return ir_map_and_lookup; +} + +// fun map.replaceIfExists(mutate self, key: K, value: V): bool +std::vector generate_mapKV_replace(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2], true); + + std::vector ir_map_and_was_added = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), loc, "(map-and-was-added)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_was_added, std::move(dict_args), lookup_function("__dict.replace")); + + return ir_map_and_was_added; +} + +// fun map.replaceAndGetPrevious(mutate self, key: K, value: V): MapLookupResult +std::vector generate_mapKV_replaceGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2]); + + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(map-and-lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.replaceGet")); + + return ir_map_and_lookup; +} + +// fun map.addIfNotExists(mutate self, key: K, value: V): bool +std::vector generate_mapKV_add(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2], true); + + std::vector ir_map_and_was_added = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), loc, "(map-and-was-added)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_was_added, std::move(dict_args), lookup_function("__dict.add")); + + return ir_map_and_was_added; +} + +// fun map.addOrGetExisting(mutate self, key: K, value: V): MapLookupResult +std::vector generate_mapKV_addGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + TypePtr TValue = called_f->substitutedTs->typeT_at(1); + DictKeyValue kv(code, loc, TKey, &args[1], TValue, &args[2]); + + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_value_val(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.addGet")); + + return ir_map_and_lookup; +} + +// fun map.delete(mutate self, key: K): bool +std::vector generate_mapKV_del(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_map_and_was_deleted = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataBool::create()}), loc, "(map-and-was-added)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_was_deleted, std::move(dict_args), lookup_function("__dict.del")); + + return ir_map_and_was_deleted; +} + +// fun map.deleteAndGetDeleted(mutate self, key: K): MapLookupResult +std::vector generate_mapKV_delGet(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_map_and_lookup = code.create_tmp_var(TypeDataTensor::create({TypeDataCell::create(), TypeDataSlice::create(), TypeDataInt::create()}), loc, "(lookup)"); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_map_and_lookup, std::move(dict_args), lookup_function("__dict.delGet")); + + return ir_map_and_lookup; +} + +// fun map.findFirst(): MapEntry +std::vector generate_mapKV_findFirst(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, nullptr, TypeDataSlice::create(), nullptr); + + // on a stack (ir_entry) we will have: either (x k −1) or (null null 0) + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getMin")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.findLast(): MapEntry +std::vector generate_mapKV_findLast(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, nullptr, TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getMax")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.findKeyGreater(pivotKey: K): MapEntry +std::vector generate_mapKV_findKeyGreater(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNext")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.findKeyGreaterOrEqual(pivotKey: K): MapEntry +std::vector generate_mapKV_findKeyGreaterOrEqual(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNextEq")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.findKeyLess(pivotKey: K): MapEntry +std::vector generate_mapKV_findKeyLess(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrev")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.findKeyLessOrEqual(pivotKey: K): MapEntry +std::vector generate_mapKV_findKeyLessOrEqual(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &args[1], TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrevEq")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.iterateNext(current: MapEntry): MapEntry +std::vector generate_mapKV_iterateNext(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + std::vector ir_pivot_key(args[1].begin() + 1, args[1].end() - 1); + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &ir_pivot_key, TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getNext")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +// fun map.iteratePrev(current: MapEntry): MapEntry +std::vector generate_mapKV_iteratePrev(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + std::vector ir_pivot_key(args[1].begin() + 1, args[1].end() - 1); + TypePtr TKey = called_f->substitutedTs->typeT_at(0); + DictKeyValue kv(code, loc, TKey, &ir_pivot_key, TypeDataSlice::create(), nullptr); + + std::vector ir_entry = create_ir_MapEntry(code, loc); + std::vector dict_args = { kv.ir_key_kind(), kv.ir_value_kind(), kv.ir_key_val(), args[0][0], kv.ir_key_len() }; + code.emplace_back(loc, Op::_Call, ir_entry, std::move(dict_args), lookup_function("__dict.getPrev")); + + return finalize_ir_MapEntry(code, loc, std::move(ir_entry), TKey); +} + +} // namespace tolk diff --git a/tolk/send-message-api.h b/tolk/maps-kv-api.h similarity index 52% rename from tolk/send-message-api.h rename to tolk/maps-kv-api.h index f86be5446..d6d64adc4 100644 --- a/tolk/send-message-api.h +++ b/tolk/maps-kv-api.h @@ -16,16 +16,12 @@ */ #pragma once +#include "fwd-declarations.h" #include "tolk.h" namespace tolk { -std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect); -std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect); - -std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options); - -std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy); -std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy, std::vector&& ir_address); +bool check_mapKV_TKey_is_valid(TypePtr TKey, std::string& because_msg); +bool check_mapKV_TValue_is_valid(TypePtr TValue, std::string& because_msg); } // namespace tolk diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 93c6e2067..a3a0e2916 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -422,6 +422,43 @@ bool Optimizer::detect_rewrite_NOT_THROWIF() { return false; } +// pattern `NEWC` + store const slice + XCHG + keyLen + DICTSETB -> push const slice + XCHG + keyLen + DICTSET +// (useful for `someMap.set(k, constVal)` where constVal is represented as a const slice) +bool Optimizer::detect_rewrite_DICTSETB_DICTSET() { + bool fifth_dict = pb_ >= 5 && op_[4]->is_custom() && op_[4]->op.starts_with("DICT"); + if (!fifth_dict) { + return false; + } + + bool first_newc = op_[0]->op == "NEWC"; + bool second_stsliceconst = op_[1]->op.ends_with(" STSLICECONST"); + bool third_xchg = op_[2]->is_xchg() || op_[2]->op == "ROT" || op_[2]->op == "-ROT" || op_[2]->op.ends_with(" PUXC"); + bool fourth_pushint = op_[3]->is_const() && op_[3]->op.ends_with(" PUSHINT"); + if (!first_newc || !second_stsliceconst || !third_xchg || !fourth_pushint) { + return false; + } + + static const char* contains_b[] = {"SETB", "REPLACEB", "ADDB", "GETB"}; + static const char* repl_with[] = {"SET", "REPLACE", "ADD", "GET" }; + + std::string new_op = op_[4]->op; // "DICTSET" / "DICTSETGET NULLSWAPIFNOT" + for (size_t i = 0; i < std::size(contains_b); ++i) { + if (size_t pos = new_op.find(contains_b[i]); pos == 4 || pos == 5) { + new_op.replace(pos, std::strlen(contains_b[i]), repl_with[i]); + p_ = 5; + q_ = 4; + oq_[0] = std::make_unique(AsmOp::Custom(op_[1]->loc, op_[1]->op.substr(0, op_[1]->op.rfind(' ')) + " PUSHSLICE", 0, 1)); + oq_[1] = std::move(op_[2]); + oq_[2] = std::move(op_[3]); + oq_[3] = std::make_unique(AsmOp::Custom(op_[4]->loc, new_op)); + return true; + } + } + + return false; +} + + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); } @@ -844,7 +881,7 @@ bool Optimizer::find_at_least(int pb) { detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || detect_rewrite_LDxx_DROP() || detect_rewrite_SWAP_symmetric() || detect_rewrite_SWAP_PUSH_STUR() || detect_rewrite_SWAP_STxxxR() || - detect_rewrite_NOT_THROWIF() || + detect_rewrite_NOT_THROWIF() || detect_rewrite_DICTSETB_DICTSET() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/overload-resolution.cpp b/tolk/overload-resolution.cpp new file mode 100644 index 000000000..ed98bf250 --- /dev/null +++ b/tolk/overload-resolution.cpp @@ -0,0 +1,236 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "overload-resolution.h" +#include "type-system.h" +#include "compiler-state.h" + +/* + * Find an exact method having a receiver type. + * + * Given: int.copy, T.copy, Container.copy + * > 5.copy(); // 1 + * > (5 as int8).copy(); // 2 with T=int8 + * > containerOfInt.copy(); // 3 with T=int + * > nullableContainerOfInt.copy(); // 2 with T=Container? + * + */ + +namespace tolk { + +// each next shape kind is more specific than another; +// e.g., between `T.copy` and `int.copy` we choose the second; +enum class ShapeKind { + GenericT, // T + Union, // U|V, T? + Primitive, // int, slice, address, ... + Tensor, // (A,B,...) + Instantiated, // Map, Container, Struct, ... +}; + +// for every receiver, we calculate "score": how deep and specific it is; +// e.g., between `Container` and `T` we choose the first; +// e.g., between `map` and `map>` we choose the second; +struct ShapeScore { + ShapeKind kind; + int depth; + + bool is_shape_better_than(const ShapeScore& rhs) const { + if (kind != rhs.kind) { + return kind > rhs.kind; + } + return depth > rhs.depth; + } + + bool operator==(const ShapeScore& rhs) const = default; +}; + +// calculate score for a receiver; +// note: it's an original receiver, with generics, not an instantiated one +static ShapeScore calculate_shape_score(TypePtr t) { + if (t->try_as()) { + return {ShapeKind::GenericT, 1}; + } + + if (const auto* t_union = t->try_as()) { + int d = 0; + for (TypePtr variant : t_union->variants) { + d = std::max(d, calculate_shape_score(variant).depth); + } + return {ShapeKind::Union, 1 + d}; + } + + if (const auto* t_tensor = t->try_as()) { + int d = 0; + for (TypePtr item : t_tensor->items) { + d = std::max(d, calculate_shape_score(item).depth); + } + return {ShapeKind::Tensor, 1 + d}; + } + + if (const auto* t_brackets = t->try_as()) { + int d = 0; + for (TypePtr item : t_brackets->items) { + d = std::max(d, calculate_shape_score(item).depth); + } + return {ShapeKind::Tensor, 1 + d}; + } + + if (const auto* t_instTs = t->try_as()) { + int d = 0; + for (TypePtr typeT : t_instTs->type_arguments) { + d = std::max(d, calculate_shape_score(typeT).depth); + } + return {ShapeKind::Instantiated, 1 + d}; + } + + if (const auto* t_map = t->try_as()) { + int d = std::max(calculate_shape_score(t_map->TKey).depth, calculate_shape_score(t_map->TValue).depth); + return {ShapeKind::Instantiated, 1 + d}; + } + + if (const auto* t_alias = t->try_as()) { + return calculate_shape_score(t_alias->underlying_type); + } + + return {ShapeKind::Primitive, 1}; +} + +// tries to find Ts in `pattern` to reach `actual`; +// example: pattern=`map`, actual=`map` => T=int +// example: pattern=`Container`, actual=`Container>` => T=Container +static bool can_substitute_Ts_to_reach_actual(TypePtr pattern, TypePtr actual, const GenericsDeclaration* genericTs) { + GenericSubstitutionsDeducing deducingTs(genericTs); + TypePtr replaced = deducingTs.auto_deduce_from_argument(pattern, actual); + return replaced->equal_to(actual); +} + +// checks whether a generic typeA is more specific than typeB; +// example: `map` dominates `map`; +// example: `map>` dominates `map>` dominates `map>`; +// example: `map` and `map` are not comparable; +static bool is_more_specific_generic(TypePtr typeA, TypePtr typeB, const GenericsDeclaration* genericTsA, const GenericsDeclaration* genericTsB) { + // exists θ: θ(B)=A && not exists φ: φ(A)=B + return can_substitute_Ts_to_reach_actual(typeB, typeA, genericTsB) + && !can_substitute_Ts_to_reach_actual(typeA, typeB, genericTsA); +} + +// the main "overload resolution" entrypoint: given `obj.method()`, find best applicable methods; +// if there are many (no one is better than others), a caller side will emit "ambiguous call" +std::vector resolve_methods_for_call(TypePtr provided_receiver, std::string_view called_name) { + // find all methods theoretically applicable; we'll filter them by priority; + // for instance, if there is `T.method`, it will be instantiated with T=provided_receiver + std::vector viable; + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name) { + TypePtr receiver = method_ref->receiver_type; + if (receiver->has_genericT_inside()) { + try { // check whether exist some T to make it a valid call (probably with type coercion) + GenericSubstitutionsDeducing deducingTs(method_ref); + TypePtr replaced = deducingTs.auto_deduce_from_argument(receiver, provided_receiver); + if (replaced->can_rhs_be_assigned(provided_receiver)) { + viable.emplace_back(receiver, replaced, method_ref, deducingTs.flush()); + } + } catch (...) {} + } else if (receiver->can_rhs_be_assigned(provided_receiver)) { + viable.emplace_back(receiver, receiver, method_ref, GenericsSubstitutions(method_ref->genericTs)); + } + } + } + // if nothing found, return nothing; + // if the only found, it's the one + if (viable.size() <= 1) { + return viable; + } + // okay, we have multiple viable methods, and need to locate the better + + // 1) exact match candidates with equal_to() + // (for instance, an alias equals to its underlying type, as well as `T1|T2` equals to `T2|T1`) + std::vector exact; + for (const MethodCallCandidate& candidate : viable) { + if (candidate.instantiated_receiver->equal_to(provided_receiver)) { + exact.push_back(candidate); + } + } + if (exact.size() == 1) { + return exact; + } + if (!exact.empty()) { + viable = std::move(exact); + } + + // 2) if there are both generic and non-generic functions, filter out generic + size_t n_generics = 0; + for (const MethodCallCandidate& candidate : viable) { + n_generics += candidate.is_generic(); + } + if (n_generics < viable.size()) { + std::vector non_generic; + for (const MethodCallCandidate& candidate : viable) { + if (!candidate.is_generic()) { + non_generic.push_back(candidate); + } + } + // all the code below is dedicated to choosing between generic Ts, so return if non-generic + return non_generic; + } + + // 3) better shape in terms of structural depth + // (prefer `Container` over `T` and `map>` over `map`) + ShapeScore best_shape = {ShapeKind::GenericT, -999}; + for (const MethodCallCandidate& candidate : viable) { + ShapeScore s = calculate_shape_score(candidate.original_receiver); + if (s.is_shape_better_than(best_shape)) { + best_shape = s; + } + } + + std::vector best_by_shape; + for (const MethodCallCandidate& candidate : viable) { + if (calculate_shape_score(candidate.original_receiver) == best_shape) { + best_by_shape.push_back(candidate); + } + } + if (best_by_shape.size() == 1) { + return best_by_shape; + } + if (!best_by_shape.empty()) { + viable = std::move(best_by_shape); + } + + // 4) find the overload that dominates all others + // (prefer `Container` over `Container` and `map` over `map`) + const MethodCallCandidate* dominator = nullptr; + for (const MethodCallCandidate& candidate : viable) { + bool dominates_all = true; + for (const MethodCallCandidate& other : viable) { + if (candidate.method_ref != other.method_ref) { + dominates_all &= is_more_specific_generic(candidate.original_receiver, other.original_receiver, candidate.method_ref->genericTs, other.method_ref->genericTs); + } + } + if (dominates_all) { + assert(!dominator); + dominator = &candidate; + } + } + + if (dominator) { + return {*dominator}; + } + return viable; +} + +} // namespace tolk diff --git a/tolk/overload-resolution.h b/tolk/overload-resolution.h new file mode 100644 index 000000000..6960604bb --- /dev/null +++ b/tolk/overload-resolution.h @@ -0,0 +1,45 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include +#include "fwd-declarations.h" +#include "generics-helpers.h" + +namespace tolk { + +// when there are many methods with the same name, the overload resolution mechanism +// analyzes possible candidates to resolve the best match +struct MethodCallCandidate { + TypePtr original_receiver; + TypePtr instantiated_receiver; + FunctionPtr method_ref; + GenericsSubstitutions substitutedTs; + + MethodCallCandidate(TypePtr original_receiver, TypePtr instantiated_receiver, FunctionPtr method_ref, GenericsSubstitutions&& substitutedTs) + : original_receiver(original_receiver) + , instantiated_receiver(instantiated_receiver) + , method_ref(method_ref) + , substitutedTs(std::move(substitutedTs)) {} + + bool is_generic() const { return original_receiver != instantiated_receiver; } +}; + + +std::vector resolve_methods_for_call(TypePtr provided_receiver, std::string_view called_name); + +} // namespace tolk diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index abab637e9..f710d2e30 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -96,6 +96,9 @@ class PackUnpackAvailabilityChecker { if (any_type == TypeDataNever::create()) { return {}; } + if (any_type->try_as()) { + return {}; + } if (const auto* t_struct = any_type->try_as()) { StructPtr struct_ref = t_struct->struct_ref; @@ -118,6 +121,13 @@ class PackUnpackAvailabilityChecker { return {}; } + if (const auto* t_enum = any_type->try_as()) { + if (t_enum->enum_ref->members.empty()) { + return CantSerializeBecause("because `enum` is empty"); + } + return {}; + } + if (const auto* t_union = any_type->try_as()) { // a union can almost always be serialized if every of its variants can: // - `T?` is TL/B `(Maybe T)` @@ -237,15 +247,16 @@ static int calc_offset_on_stack(StructPtr struct_ref, int field_idx) { // -std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj, const std::vector& ir_options) { +// fun T.toCell(self, options: PackOptions): Cell +std::vector generate_T_toCell(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); FunctionPtr f_beginCell = lookup_function("beginCell"); FunctionPtr f_endCell = lookup_function("builder.endCell"); std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, f_beginCell); - tolk_assert(ir_options.size() == 1); // struct PackOptions - PackContext ctx(code, loc, rvect_builder, ir_options); - ctx.generate_pack_any(any_type, std::move(ir_obj)); + PackContext ctx(code, loc, rvect_builder, args[1]); + ctx.generate_pack_any(typeT, std::vector(args[0])); std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); code.emplace_back(loc, Op::_Call, rvect_cell, std::move(rvect_builder), f_endCell); @@ -253,54 +264,70 @@ std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation return rvect_cell; } -std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj, const std::vector& ir_options) { - PackContext ctx(code, loc, ir_builder, ir_options); // mutate this builder - ctx.generate_pack_any(any_type, std::move(ir_obj)); +// fun builder.storeAny(mutate self, v: T, options: PackOptions = {}): self +std::vector generate_builder_storeAny(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + PackContext ctx(code, loc, args[0], args[2]); // mutate this builder + ctx.generate_pack_any(typeT, std::vector(args[1])); - return ir_builder; // return mutated builder + return args[0]; // return mutated builder } -std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice, const std::vector& ir_options) { - if (!mutate_slice) { - std::vector slice_copy = code.create_var(TypeDataSlice::create(), loc, "s"); - code.emplace_back(loc, Op::_Let, slice_copy, std::move(ir_slice)); - ir_slice = std::move(slice_copy); - } +// fun T.fromSlice(rawSlice: slice, options: UnpackOptions): T +std::vector generate_T_fromSlice(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + std::vector slice_copy = code.create_var(TypeDataSlice::create(), loc, "s"); + code.emplace_back(loc, Op::_Let, slice_copy, args[0]); - tolk_assert(ir_options.size() == 2); // struct UnpackOptions - UnpackContext ctx(code, loc, std::move(ir_slice), ir_options); - std::vector rvect_struct = ctx.generate_unpack_any(any_type); - tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + UnpackContext ctx(code, loc, std::move(slice_copy), args[1]); + std::vector rvect_struct = ctx.generate_unpack_any(typeT); + tolk_assert(typeT->get_width_on_stack() == static_cast(rvect_struct.size())); - // slice.loadAny() ignores options.assertEndAfterReading, because it's intended to read data in the middle - if (!mutate_slice && !estimate_serialization_size(any_type).is_unpredictable_infinity()) { + if (!estimate_serialization_size(typeT).is_unpredictable_infinity()) { ctx.assertEndIfOption(); } return rvect_struct; } -std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell, const std::vector& ir_options) { +// fun slice.loadAny(mutate self, options: UnpackOptions): T +std::vector generate_slice_loadAny(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + UnpackContext ctx(code, loc, args[0], args[1]); + std::vector rvect_struct = ctx.generate_unpack_any(typeT); + tolk_assert(typeT->get_width_on_stack() == static_cast(rvect_struct.size())); + + // slice.loadAny() ignores options.assertEndAfterReading, because it's intended to read data in the middle + std::vector ir_slice_and_result = args[0]; + ir_slice_and_result.insert(ir_slice_and_result.end(), rvect_struct.begin(), rvect_struct.end()); + return ir_slice_and_result; +} + +// fun T.fromCell(packedCell: cell, options: UnpackOptions): T +// fun Cell.load(self, options: UnpackOptions): T +std::vector generate_T_fromCell(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); FunctionPtr f_beginParse = lookup_function("cell.beginParse"); std::vector ir_slice = code.create_var(TypeDataSlice::create(), loc, "s"); - code.emplace_back(loc, Op::_Call, ir_slice, std::move(ir_cell), f_beginParse); + code.emplace_back(loc, Op::_Call, ir_slice, args[0], f_beginParse); - tolk_assert(ir_options.size() == 2); // struct UnpackOptions - UnpackContext ctx(code, loc, std::move(ir_slice), ir_options); - std::vector rvect_struct = ctx.generate_unpack_any(any_type); - tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + UnpackContext ctx(code, loc, std::move(ir_slice), args[1]); + std::vector rvect_struct = ctx.generate_unpack_any(typeT); + tolk_assert(typeT->get_width_on_stack() == static_cast(rvect_struct.size())); // if a struct has RemainingBitsAndRefs, don't test it for assertEnd - if (!estimate_serialization_size(any_type).is_unpredictable_infinity()) { + if (!estimate_serialization_size(typeT).is_unpredictable_infinity()) { ctx.assertEndIfOption(); } return rvect_struct; } -std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, const std::vector& ir_options) { - UnpackContext ctx(code, loc, ir_slice, ir_options); // mutate this slice - ctx.generate_skip_any(any_type); +// fun slice.skipAny(mutate self, options: UnpackOptions): self +std::vector generate_slice_skipAny(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + UnpackContext ctx(code, loc, args[0], args[1]); // mutate this slice + ctx.generate_skip_any(typeT); - return ir_slice; // return mutated slice + return args[0]; // return mutated slice } void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj) { @@ -404,6 +431,9 @@ std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation std::vector generate_lazy_match_for_union(CodeBlob& code, SrcLocation loc, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options) { tolk_assert(lazy_variable->ir_options.size() == 2); + if (options.match_blocks.empty()) { // empty `match` statement, no arms + return {}; + } UnpackContext ctx(code, loc, lazy_variable->ir_slice, lazy_variable->ir_options); std::vector rvect_match = ctx.generate_lazy_match_any(union_type, options); @@ -423,14 +453,19 @@ std::vector generate_lazy_object_finish_loading(CodeBlob& code, SrcLo return lazy_variable->ir_slice; } +std::vector generate_T_forceLoadLazyObject(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + // a call to `T.forceLoadLazyObject()` was handled separately + tolk_assert(false); +} + PackSize estimate_serialization_size(TypePtr any_type) { EstimateContext ctx; return ctx.estimate_any(any_type); } -std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type) { - EstimateContext ctx; - PackSize pack_size = ctx.estimate_any(any_type); +std::vector generate_T_estimatePackSize(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + PackSize pack_size = estimate_serialization_size(typeT); std::vector ir_tensor = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), loc, "(result-tensor)"); code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[0]}, td::make_refint(pack_size.min_bits)); @@ -438,11 +473,7 @@ std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation l code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[2]}, td::make_refint(pack_size.min_refs)); code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[3]}, td::make_refint(pack_size.max_refs)); - FunctionPtr f_toTuple = lookup_function("T.__toTuple"); - std::vector ir_tuple = code.create_tmp_var(TypeDataTuple::create(), loc, "(result-tuple)"); - code.emplace_back(loc, Op::_Call, ir_tuple, ir_tensor, f_toTuple); - - return ir_tuple; + return ir_tensor; } } // namespace tolk diff --git a/tolk/pack-unpack-api.h b/tolk/pack-unpack-api.h index 7b4d02f66..45da429e5 100644 --- a/tolk/pack-unpack-api.h +++ b/tolk/pack-unpack-api.h @@ -22,15 +22,10 @@ namespace tolk { bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg); - -std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj, const std::vector& ir_options); -std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj, const std::vector& ir_options); -std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice, const std::vector& ir_options); -std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell, const std::vector& ir_options); -std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, const std::vector& ir_options); - PackSize estimate_serialization_size(TypePtr any_type); -std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type); + +// functions like T.toCell() are not declared here: they are implemented in a .cpp file, +// and their prototypes exist and are referenced in `builtins.cpp` struct LazyStructLoadInfo; struct LazyStructLoadedState; diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 5f1ebcc8f..d000bfb6f 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -210,7 +210,7 @@ void UnpackContext::throwInvalidOpcode() const { const LazyMatchOptions::MatchBlock* LazyMatchOptions::find_match_block(TypePtr variant) const { for (const MatchBlock& b : match_blocks) { - if (b.arm_variant->get_type_id() == variant->get_type_id()) { + if (b.arm_variant->equal_to(variant)) { return &b; } } @@ -1056,6 +1056,85 @@ struct S_CustomStruct final : ISerializer { } }; +struct S_IntegerEnum final : ISerializer { + EnumDefPtr enum_ref; + + explicit S_IntegerEnum(EnumDefPtr enum_ref) + : enum_ref(enum_ref) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + TypePtr intN = calculate_intN_to_serialize_enum(enum_ref); + return ctx->generate_pack_any(intN, std::move(rvect)); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + const TypeDataIntN* intN = calculate_intN_to_serialize_enum(enum_ref)->try_as(); + std::vector ir_num = ctx->generate_unpack_any(intN); + + // when reading an integer value, we need to validate that it's a valid enum member; + // at first, detect whether it's a sequence (A, A+1, ..., A+N) + bool is_sequence = true; + td::RefInt256 expected_cur = enum_ref->members.front()->computed_value; + for (EnumMemberPtr member_ref : enum_ref->members) { + is_sequence &= td::cmp(member_ref->computed_value, expected_cur) == 0; + expected_cur += 1; + } + + if (is_sequence) { + // enum's members are A...B one by one (probably, 0...M); + // then validation is: "throw if vB", but "LESSINT + THROWIF" 2 times is more generalized + td::RefInt256 min_value = enum_ref->members.front()->computed_value; + bool dont_check_min = intN->is_unsigned && min_value == 0; + if (!dont_check_min) { // LDU can't load < 0 + std::vector ir_min_value = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-min)"); + code.emplace_back(loc, Op::_IntConst, ir_min_value, min_value); + std::vector ir_lt_min = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-lt-min)"); + code.emplace_back(loc, Op::_Call, ir_lt_min, std::vector{ir_num[0], ir_min_value[0]}, lookup_function("_<_")); + std::vector args_assert1 = { code.create_int(loc, 5, "(excno)"), ir_lt_min[0], code.create_int(loc, 1, "") }; + Op& op_assert1 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert1), lookup_function("__throw_if_unless")); + op_assert1.set_impure_flag(); + } + td::RefInt256 max_value = enum_ref->members.back()->computed_value; + bool dont_check_max = intN->is_unsigned && max_value == (1ULL << intN->n_bits) - 1; + if (!dont_check_max) { // LDU can't load >= 1<_")); + std::vector args_assert2 = { code.create_int(loc, 5, "(excno)"), ir_gt_max[0], code.create_int(loc, 1, "") }; + Op& op_assert2 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert2), lookup_function("__throw_if_unless")); + op_assert2.set_impure_flag(); + } + } else { + // okay, enum is not a sequence, just a set of values; + // then validation is: "throw if v is not contained in V", check v==V_i and combine with OR + var_idx_t ir_any_of = code.create_int(loc, 0, "(any-of-equals)"); + for (EnumMemberPtr member_ref : enum_ref->members) { + std::vector ir_ith_value = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-ith)"); + code.emplace_back(loc, Op::_IntConst, ir_ith_value, member_ref->computed_value); + std::vector ir_ith_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(enum-ith-eq)"); + code.emplace_back(loc, Op::_Call, ir_ith_eq, std::vector{ir_num[0], ir_ith_value[0]}, lookup_function("_==_")); + code.emplace_back(loc, Op::_Call, std::vector{ir_any_of}, std::vector{ir_any_of, ir_ith_eq[0]}, lookup_function("_|_")); + } + std::vector args_assert = { code.create_int(loc, 5, "(excno)"), ir_any_of, code.create_int(loc, 0, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); + } + return ir_num; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + TypePtr intN = calculate_intN_to_serialize_enum(enum_ref); + return ctx->generate_skip_any(intN); + } + + PackSize estimate(const EstimateContext* ctx) override { + TypePtr intN = calculate_intN_to_serialize_enum(enum_ref); + return ctx->estimate_any(intN); + } +}; + struct S_CustomReceiverForPackUnpack final : ISerializer { TypePtr receiver_type; @@ -1163,6 +1242,62 @@ std::vector auto_generate_opcodes_for_union(TypePtr union_type, std: return result; } +// given a `enum`, calculate N bits enough to store all values; +// example: `enum Color { Red, Green, Blue }` is 00/01/10 - uint2 +// example: `enum Role: int8 { ... }` — manually specified +TypePtr calculate_intN_to_serialize_enum(EnumDefPtr enum_ref) { + if (enum_ref->colon_type) { + return enum_ref->colon_type; + } + + bool is_unsigned = false; + int n_bits = 1; + for (; n_bits <= 256; ++n_bits) { + bool fits_unsigned = std::all_of(enum_ref->members.begin(), enum_ref->members.end(), [n_bits](EnumMemberPtr member_ref) { + return member_ref->computed_value->unsigned_fits_bits(n_bits); + }); + if (fits_unsigned) { + is_unsigned = true; + break; + } + bool fits_signed = std::all_of(enum_ref->members.begin(), enum_ref->members.end(), [n_bits](EnumMemberPtr member_ref) { + return member_ref->computed_value->signed_fits_bits(n_bits); + }); + if (fits_signed) { + break; + } + } + + return TypeDataIntN::create(n_bits, is_unsigned, false); +} + +// there is no way to pass custom pack options to createMessage / map.set / etc., using hardcoded ones +std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc) { + StructPtr s_PackOptions = lookup_global_symbol("PackOptions")->try_as(); + std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_PackOptions), loc, "(pack-options)"); + tolk_assert(ir_options.size() == 1); + + std::vector ir_defaults = { + code.create_int(loc, 0, "(zero)"), // skipBitsNFieldsValidation + }; + code.emplace_back(loc, Op::_Let, ir_options, std::move(ir_defaults)); + return ir_options; +} + +// there is no way to pass custom unpack options to map.get / etc., using hardcoded ones +std::vector create_default_UnpackOptions(CodeBlob& code, SrcLocation loc) { + StructPtr s_UnpackOptions = lookup_global_symbol("UnpackOptions")->try_as(); + std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_UnpackOptions), loc, "(unpack-options)"); + tolk_assert(ir_options.size() == 2); + + std::vector ir_defaults = { + code.create_int(loc, -1, "(true)"), // assertEndAfterReading + code.create_int(loc, 63, "(excno)"), // throwIfOpcodeDoesNotMatch + }; + code.emplace_back(loc, Op::_Let, ir_options, std::move(ir_defaults)); + return ir_options; +} + // -------------------------------------------- // detect serializer by TypePtr @@ -1207,9 +1342,15 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { return std::make_unique(); } + if (any_type->try_as()) { + return std::make_unique(); + } if (const auto* t_struct = any_type->try_as()) { return std::make_unique(t_struct->struct_ref); } + if (const auto* t_enum = any_type->try_as()) { + return std::make_unique(t_enum->enum_ref); + } if (const auto* t_union = any_type->try_as()) { // `T?` is always `(Maybe T)`, even if T has custom opcode (opcode will follow bit '1') diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index ab11733aa..a511f751e 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -165,5 +165,9 @@ class EstimateContext { bool is_type_cellT(TypePtr any_type); FunctionPtr get_custom_pack_unpack_function(TypePtr receiver_type, bool is_pack); std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg); +TypePtr calculate_intN_to_serialize_enum(EnumDefPtr enum_ref); + +std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc); +std::vector create_default_UnpackOptions(CodeBlob& code, SrcLocation loc); } // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index f22b0ab44..568f4868d 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -24,9 +24,6 @@ #include "smart-casts-cfg.h" #include "pack-unpack-api.h" #include "gen-entrypoints.h" -#include "generics-helpers.h" -#include "send-message-api.h" -#include "gen-entrypoints.h" /* * This pipe is the last one operating AST: it transforms AST to IR. @@ -386,6 +383,15 @@ const LazyVariableLoadedState* CodeBlob::get_lazy_variable(AnyExprV v) const { return nullptr; } +const CachedConstValueAtCodegen* CodeBlob::get_cached_const(GlobalConstPtr const_ref) const { + for (const CachedConstValueAtCodegen& c : cached_consts) { + if (c.const_ref == const_ref) { + return &c; + } + } + return nullptr; +} + // given `{some_expr}!`, return some_expr static AnyExprV unwrap_not_null_operator(AnyExprV v) { @@ -573,7 +579,7 @@ std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, Ty FunctionPtr not_sym = lookup_function("!b_"); std::vector result_ir_idx = code.create_tmp_var(TypeDataBool::create(), loc, debug_desc); - const TypeDataUnion* lhs_union = expr_type->try_as(); + const TypeDataUnion* lhs_union = expr_type->unwrap_alias()->try_as(); if (!lhs_union) { // `int` is `int` / `int` is `builder`, it's compile-time, either 0, or -1 bool types_eq = expr_type->get_type_id() == cmp_type->get_type_id(); @@ -620,8 +626,6 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob if (called_f->is_method() && called_f->is_instantiation_of_generic_function()) { std::string_view f_name = called_f->base_fun_ref->name; - TypePtr typeT = called_f->substitutedTs->typeT_at(0); - const LazyVariableLoadedState* lazy_variable = v_call->dot_obj_is_self ? code.get_lazy_variable(v_call->get_self_obj()) : nullptr; if (f_name == "T.toCell" && lazy_variable && lazy_variable->is_struct()) { @@ -629,49 +633,6 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob std::vector ir_obj = vars_per_arg[0]; // = lazy_var_ref->ir_idx return generate_lazy_struct_to_cell(code, loc, &lazy_variable->loaded_state, std::move(ir_obj), vars_per_arg[1]); } - if (f_name == "T.toCell") { - // in: object T, out: Cell (just a cell, wrapped) - std::vector ir_obj = vars_per_arg[0]; - return generate_pack_struct_to_cell(code, loc, typeT, std::move(ir_obj), vars_per_arg[1]); - } - if (f_name == "T.fromCell") { - // in: cell, out: object T - std::vector ir_cell = vars_per_arg[0]; - return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell), vars_per_arg[1]); - } - if (f_name == "T.fromSlice") { - // in: slice, out: object T, input slice NOT mutated - std::vector ir_slice = vars_per_arg[0]; - return generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), false, vars_per_arg[1]); - } - if (f_name == "Cell.load") { - // in: cell, out: object T - std::vector ir_cell = vars_per_arg[0]; - return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell), vars_per_arg[1]); - } - if (f_name == "slice.loadAny") { - // in: slice, out: object T, input slice is mutated, so prepend self before an object - var_idx_t ir_self = vars_per_arg[0][0]; - std::vector ir_slice = vars_per_arg[0]; - std::vector ir_obj = generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), true, vars_per_arg[1]); - std::vector ir_result = {ir_self}; - ir_result.insert(ir_result.end(), ir_obj.begin(), ir_obj.end()); - return ir_result; - } - if (f_name == "slice.skipAny") { - // in: slice, out: the same slice, with a shifted pointer - std::vector ir_slice = vars_per_arg[0]; - return generate_skip_struct_in_slice(code, loc, typeT, std::move(ir_slice), vars_per_arg[1]); - } - if (f_name == "builder.storeAny") { - // in: builder and object T, out: mutated builder - std::vector ir_builder = vars_per_arg[0]; - std::vector ir_obj = vars_per_arg[1]; - return generate_pack_struct_to_builder(code, loc, typeT, std::move(ir_builder), std::move(ir_obj), vars_per_arg[2]); - } - if (f_name == "T.estimatePackSize") { - return generate_estimate_size_call(code, loc, typeT); - } if (f_name == "T.forceLoadLazyObject") { // in: object T, out: slice (same slice that a lazy variable holds, after loading/skipping all its fields) if (!lazy_variable) { @@ -682,36 +643,9 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob } } - if (called_f->is_instantiation_of_generic_function()) { - std::string_view f_name = called_f->base_fun_ref->name; - TypePtr typeT = called_f->substitutedTs->typeT_at(0); - - if (f_name == "createMessage") { - std::vector ir_msg_params = vars_per_arg[0]; - return generate_createMessage(code, loc, typeT->unwrap_alias(), std::move(ir_msg_params)); - } - if (f_name == "createExternalLogMessage") { - std::vector ir_msg_params = vars_per_arg[0]; - return generate_createExternalLogMessage(code, loc, typeT->unwrap_alias(), std::move(ir_msg_params)); - } - } - - if (called_f->name == "address.buildSameAddressInAnotherShard") { - std::vector ir_self_address = vars_per_arg[0]; - std::vector ir_shard_options = vars_per_arg[1]; - return generate_address_buildInAnotherShard(code, loc, std::move(ir_self_address), std::move(ir_shard_options)); - } - if (called_f->name == "AutoDeployAddress.buildAddress") { - std::vector ir_self = vars_per_arg[0]; - return generate_AutoDeployAddress_buildAddress(code, loc, std::move(ir_self)); - } - if (called_f->name == "AutoDeployAddress.addressMatches") { - std::vector ir_self = vars_per_arg[0]; - std::vector ir_address = vars_per_arg[1]; - return generate_AutoDeployAddress_addressMatches(code, loc, std::move(ir_self), std::move(ir_address)); - } - - tolk_assert(false); + auto gen = std::get_if(&called_f->body); + tolk_assert(gen); + return (*gen)->generate_ops(called_f, code, loc, vars_per_arg); } std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg) { @@ -786,12 +720,12 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector(rvect.size()) == original_type->get_width_on_stack()); #endif + // aliases are erased at the TVM level original_type = original_type->unwrap_alias(); target_type = target_type->unwrap_alias(); // pass `T` to `T` - // could occur for passing tensor `(..., T, ...)` to `(..., T, ...)` while traversing tensor's components - if (target_type == original_type) { + if (target_type->equal_to(original_type)) { return rvect; } @@ -834,7 +768,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoris_primitive_nullable() && orig_w > 0) { // nothing except "T1 | T2 | ... null" can be cast to 1-slot nullable `T1?` - tolk_assert(o_union && o_union->has_null() && o_union->has_variant_with_type_id(t_union->or_null)); + tolk_assert(o_union && o_union->has_null() && o_union->has_variant_equal_to(t_union->or_null)); // here we exploit rvect shape, how union types and multi-slot nullables are stored on a stack // `T1 | T2 | ... | null` occupies N+1 slots, where the last is for UTag // when it holds null value, N slots are null, and UTag slot is 0 (it's type_id of TypeDataNullLiteral) @@ -900,7 +834,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoris_primitive_nullable() && t_union) { - tolk_assert(t_union->has_null() && t_union->has_variant_with_type_id(o_union->or_null) && target_w > 1); + tolk_assert(t_union->has_null() && t_union->has_variant_equal_to(o_union->or_null) && target_w > 1); // the transformation is tricky: // when value is null, we need to achieve "... (null) 0" (value is already null, so "... value 0") // when value is not null, we need to get "... value {type_id}" @@ -1075,6 +1009,16 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { + return rvect; + } + + // pass `address` to `bits267` + if (original_type == TypeDataAddress::create() && target_type->try_as()) { + return rvect; + } + // pass a typed tuple `[int, int]` to an untyped (via `as` operator) if (original_type->try_as() && target_type->try_as()) { return rvect; @@ -1158,6 +1102,18 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { + return rvect; + } + if (original_type == TypeDataInt::create() && target_type->try_as()) { + return rvect; + } + // `Color.Red` as `BounceMode` (all enums are integers, they can be cast one to another) + if (original_type->try_as() && target_type->try_as()) { + return rvect; + } + throw Fatal("unhandled transition_expr_to_runtime_type_impl() combination"); } @@ -1186,6 +1142,38 @@ std::vector transition_to_target_type(std::vector&& rvect, std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, CodeBlob& code, LValContext* lval_ctx) { + // referencing a local variable (not its declaration, but its usage) + if (LocalVarPtr var_ref = sym->try_as()) { +#ifdef TOLK_DEBUG + tolk_assert(static_cast(var_ref->ir_idx.size()) == var_ref->declared_type->get_width_on_stack()); +#endif + return var_ref->ir_idx; + } + + // referencing a global constant, embed its init_value directly (so, it will be evaluated to a const val later) + if (GlobalConstPtr const_ref = sym->try_as()) { + tolk_assert(lval_ctx == nullptr); + // when evaluating `a1` in `const a2 = a1 + a1`, cache a1's IR vars to prevent exponential explosion + if (code.inside_evaluating_constant) { + if (const CachedConstValueAtCodegen* cached = code.get_cached_const(const_ref)) { + return cached->ir_idx; + } + std::vector ir_sub_const = pre_compile_expr(const_ref->init_value, code, const_ref->declared_type); + code.cached_consts.emplace_back(CachedConstValueAtCodegen{const_ref, ir_sub_const}); + return ir_sub_const; + } + // just referencing `a1` in a function body + // (note that `a1` occurred 2 times has 2 different ir_idx, caching above is done only within another const) + ASTAuxData* aux_data = new AuxData_ForceFiftLocation(loc); + auto v_force_loc = createV(loc, const_ref->init_value, aux_data, const_ref->inferred_type); + code.inside_evaluating_constant = true; + std::vector ir_const = pre_compile_expr(v_force_loc, code, const_ref->declared_type); + code.inside_evaluating_constant = false; + code.cached_consts.clear(); + return ir_const; + } + + // referencing a global variable, copy it to a local tmp var if (GlobalVarPtr glob_ref = sym->try_as()) { // handle `globalVar = rhs` / `mutate globalVar` if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { @@ -1202,23 +1190,14 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co } return local_ir_idx; } - if (GlobalConstPtr const_ref = sym->try_as()) { - tolk_assert(lval_ctx == nullptr); - ASTAuxData* aux_data = new AuxData_ForceFiftLocation(loc); - auto v_force_loc = createV(loc, const_ref->init_value, aux_data, const_ref->inferred_type); - return pre_compile_expr(v_force_loc, code, const_ref->declared_type); - } + + // referencing a function (not calling it! using as a callback, works similar to a global var) if (FunctionPtr fun_ref = sym->try_as()) { std::vector rvect = code.create_tmp_var(fun_ref->inferred_full_type, loc, "(glob-var-fun)"); code.emplace_back(loc, Op::_GlobVar, rvect, std::vector{}, fun_ref); return rvect; } - if (LocalVarPtr var_ref = sym->try_as()) { -#ifdef TOLK_DEBUG - tolk_assert(static_cast(var_ref->ir_idx.size()) == var_ref->declared_type->get_width_on_stack()); -#endif - return var_ref->ir_idx; - } + throw Fatal("pre_compile_symbol"); } @@ -1356,8 +1335,8 @@ static std::vector process_cast_as_operator(V v } static std::vector process_is_type_operator(V v, CodeBlob& code, TypePtr target_type) { - TypePtr lhs_type = v->get_expr()->inferred_type->unwrap_alias(); - TypePtr cmp_type = v->type_node->resolved_type->unwrap_alias(); + TypePtr lhs_type = v->get_expr()->inferred_type; + TypePtr cmp_type = v->type_node->resolved_type; bool is_null_check = cmp_type == TypeDataNullLiteral::create(); // `v == null`, not `v is T` tolk_assert(!cmp_type->try_as()); // `v is int|slice` is a type checker error @@ -1372,7 +1351,7 @@ static std::vector process_is_type_operator(V v } static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr expr_type = v->get_expr()->inferred_type; TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type, TypeDataNullLiteral::create()); TypePtr child_target_type = without_null_type != TypeDataNever::create() ? without_null_type : expr_type; @@ -1433,7 +1412,8 @@ static std::vector process_lazy_operator(V v, Code } static std::vector process_match_expression(V v, CodeBlob& code, TypePtr target_type) { - TypePtr lhs_type = v->get_subject()->inferred_type->unwrap_alias(); + TypePtr subject_type = v->get_subject()->inferred_type; + const TypeDataEnum* subject_enum = subject_type->unwrap_alias()->try_as(); int n_arms = v->get_arms_count(); std::vector subj_ir_idx = pre_compile_expr(v->get_subject(), code, nullptr); @@ -1444,18 +1424,28 @@ static std::vector process_match_expression(V v return {}; } + bool has_type_arm = false; + bool has_expr_arm = false; + bool has_else_arm = false; + for (int i = 0; i < n_arms; ++i) { + auto v_arm = v->get_arm(i); + has_type_arm |= v_arm->pattern_kind == MatchArmKind::exact_type; + has_expr_arm |= v_arm->pattern_kind == MatchArmKind::const_expression; + has_else_arm |= v_arm->pattern_kind == MatchArmKind::else_branch; + } + // it's either `match` by type (all arms patterns are types) or `match` by expression - bool is_match_by_type = v->get_arm(0)->pattern_kind == MatchArmKind::exact_type; - bool last_else_branch = v->get_arm(n_arms - 1)->pattern_kind == MatchArmKind::else_branch; + bool is_match_by_type = has_type_arm; // detect whether `match` is exhaustive bool is_exhaustive = is_match_by_type // match by type covers all cases, checked earlier - || !v->is_statement() // match by expression is guaranteely exhaustive, checked earlier - || last_else_branch; + || !v->is_statement() // match by expression is exhaustive, checked earlier + || (has_expr_arm && subject_enum) // match by enum (either covers all cases or contains else, checked earlier) + || has_else_arm; // `else` is not allowed in `match` by type; this was not fired at type checking, // because it might turned out to be a lazy match, where `else` is allowed; // if we are here, it's not a lazy match, it's a regular one (the lazy one is handled specially, in aux vertex) - if (is_match_by_type && last_else_branch) { + if (is_match_by_type && has_else_arm) { throw ParseError(v->get_arm(n_arms - 1)->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); } @@ -1470,9 +1460,9 @@ static std::vector process_match_expression(V v auto v_ith_arm = v->get_arm(i); std::vector eq_ith_ir_idx; if (is_match_by_type) { - TypePtr cmp_type = v_ith_arm->pattern_type_node->resolved_type->unwrap_alias(); - tolk_assert(!cmp_type->try_as()); // `match` over `int|slice` is a type checker error - eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); + TypePtr cmp_type = v_ith_arm->pattern_type_node->resolved_type; + tolk_assert(!cmp_type->unwrap_alias()->try_as()); // `match` over `int|slice` is a type checker error + eq_ith_ir_idx = pre_compile_is_type(code, subject_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); } else { std::vector ith_ir_idx = pre_compile_expr(v_ith_arm->get_pattern_expr(), code); tolk_assert(subj_ir_idx.size() == 1 && ith_ir_idx.size() == 1); @@ -1518,7 +1508,7 @@ static std::vector process_match_expression(V v static std::vector process_dot_access(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { // it's NOT a method call `t.tupleSize()` (since such cases are handled by process_function_call) // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) - if (!v->is_target_fun_ref()) { + if (v->is_target_indexed_access() || v->is_target_struct_field()) { TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); // `user.id`; internally, a struct (an object) is a tensor if (const auto* t_struct = obj_type->try_as()) { @@ -1594,6 +1584,15 @@ static std::vector process_dot_access(V v, CodeBlob& } tolk_assert(false); } + // `Color.Red` + if (v->is_target_enum_member()) { + // all enums are integers, and their integer values have already been assigned or auto-calculated + EnumMemberPtr member_ref = std::get(v->target); + tolk_assert(!member_ref->computed_value.is_null()); + std::vector enum_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->get_identifier()->loc, "(enum-member)"); + code.emplace_back(v->loc, Op::_IntConst, enum_ir_idx, member_ref->computed_value); + return transition_to_target_type(std::move(enum_ir_idx), code, target_type, v); + } // okay, v->target refs a function, like `obj.method`, filled at type inferring // (currently, nothing except a global function can be referenced, no object-scope methods exist) @@ -1989,7 +1988,7 @@ static std::vector process_artificial_aux_vertex(Vget_arm(i); TypePtr arm_variant = nullptr; if (v_arm->pattern_kind == MatchArmKind::exact_type) { - arm_variant = v_arm->pattern_type_node->resolved_type->unwrap_alias(); + arm_variant = v_arm->pattern_type_node->resolved_type; } else { tolk_assert(v_arm->pattern_kind == MatchArmKind::else_branch); // `else` allowed in a lazy match } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 97a953f4f..a7bbe549f 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -191,18 +191,29 @@ static bool expect_boolean(AnyExprV v_inferred) { return expect_boolean(v_inferred->inferred_type); } -static bool expect_address(TypePtr inferred_type) { - if (inferred_type == TypeDataAddress::create()) { +static bool expect_thrown_code(TypePtr t_ex_no) { + return expect_integer(t_ex_no) || t_ex_no->unwrap_alias()->try_as(); +} + +static bool check_eq_neq_operator(TypePtr lhs_type, TypePtr rhs_type, bool& not_integer_comparison) { + if (expect_integer(lhs_type) && expect_integer(rhs_type)) { return true; } - if (const TypeDataAlias* as_alias = inferred_type->try_as()) { - return expect_address(as_alias->underlying_type); + if (expect_boolean(lhs_type) && expect_boolean(rhs_type)) { + return true; + } + if (lhs_type->equal_to(TypeDataAddress::create()) && rhs_type->equal_to(TypeDataAddress::create())) { + not_integer_comparison = true; // `address` can be compared with ==, but it's handled specially + return true; } - return false; -} -static bool expect_address(AnyExprV v_inferred) { - return expect_address(v_inferred->inferred_type); + const TypeDataEnum* lhs_enum = lhs_type->unwrap_alias()->try_as(); + const TypeDataEnum* rhs_enum = rhs_type->unwrap_alias()->try_as(); + if (lhs_enum && rhs_enum) { // allow `someColor == anotherColor`, don't allow `someColor == 123` + return lhs_enum->enum_ref == rhs_enum->enum_ref; + } + + return false; } class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { @@ -258,17 +269,16 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // (if to allow `int?`/`int8?` in the future, `==` must be expressed in a complicated Fift code considering TVM NULL) case tok_eq: case tok_neq: { - bool both_int = expect_integer(lhs) && expect_integer(rhs); - bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); - if (!both_int && !both_bool) { - bool both_address = expect_address(lhs) && expect_address(rhs); - if (both_address) { // address can be compared with ==, but it's not integer comparison, it's handled specially - v->mutate()->assign_fun_ref(nullptr); - } else if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? + bool not_integer_comparison = false; + if (!check_eq_neq_operator(lhs->inferred_type, rhs->inferred_type, not_integer_comparison)) { + if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? fire(cur_f, v->loc, "type " + to_string(lhs) + " can not be compared with `== !=`"); } else { fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); } + } + if (not_integer_comparison) { // special handling at IR generation like for `address` + v->mutate()->assign_fun_ref(nullptr); } break; } @@ -562,10 +572,12 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool has_expr_arm = false; bool has_else_arm = false; AnyExprV v_subject = v->get_subject(); - TypePtr subject_type = v_subject->inferred_type->unwrap_alias(); - const TypeDataUnion* subject_union = subject_type->try_as(); + TypePtr subject_type = v_subject->inferred_type; + const TypeDataEnum* subject_enum = subject_type->unwrap_alias()->try_as(); + const TypeDataUnion* subject_union = subject_type->unwrap_alias()->try_as(); - std::vector covered_type_ids; // for type-based `match`, what types are on the left of `=>` + std::vector covered_types; // for type-based `match`, what types are on the left of `=>` + std::vector covered_enum; // for `match` over an enum, we want it to be exhaustive for (int i = 0; i < v->get_arms_count(); ++i) { auto v_arm = v->get_arm(i); @@ -579,19 +591,22 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } has_type_arm = true; - TypePtr lhs_type = v_arm->pattern_type_node->resolved_type->unwrap_alias(); // `lhs_type => ...` - if (lhs_type->try_as()) { + TypePtr lhs_type = v_arm->pattern_type_node->resolved_type; // `lhs_type => ...` + if (lhs_type->unwrap_alias()->try_as()) { fire(cur_f, v_arm->loc, "wrong pattern matching: union types are not allowed, use concrete types in `match`"); } - bool can_happen = (subject_union && subject_union->has_variant_with_type_id(lhs_type)) || + bool can_happen = (subject_union && subject_union->has_variant_equal_to(lhs_type)) || (!subject_union && subject_type->equal_to(lhs_type)); if (!can_happen) { fire(cur_f, v_arm->loc, "wrong pattern matching: " + to_string(lhs_type) + " is not a variant of " + to_string(subject_type)); } - if (std::find(covered_type_ids.begin(), covered_type_ids.end(), lhs_type->get_type_id()) != covered_type_ids.end()) { + auto it_mentioned = std::find_if(covered_types.begin(), covered_types.end(), [lhs_type](TypePtr existing) { + return existing->equal_to(lhs_type); + }); + if (it_mentioned != covered_types.end()) { fire(cur_f, v_arm->loc, "wrong pattern matching: duplicated " + to_string(lhs_type)); } - covered_type_ids.push_back(lhs_type->get_type_id()); + covered_types.push_back(lhs_type); break; } case MatchArmKind::const_expression: { @@ -602,16 +617,26 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire(cur_f, v_arm->loc, "`else` branch should be the last"); } has_expr_arm = true; - TypePtr pattern_type = v_arm->get_pattern_expr()->inferred_type->unwrap_alias(); - bool both_int = expect_integer(pattern_type) && expect_integer(subject_type); - bool both_bool = expect_boolean(pattern_type) && expect_boolean(subject_type); - if (!both_int && !both_bool) { + TypePtr pattern_type = v_arm->get_pattern_expr()->inferred_type; + bool not_integer_comparison = false; + if (!check_eq_neq_operator(pattern_type, subject_type, not_integer_comparison) || not_integer_comparison) { if (pattern_type->equal_to(subject_type)) { // `match` over `slice` etc., where operator `==` can't be applied fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(subject_type) + " in `match`"); } else { fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(v_arm->get_pattern_expr()) + " with match subject of type " + to_string(v_subject)); } } + if (subject_enum) { + auto l_dot = v_arm->get_pattern_expr()->try_as(); + if (!l_dot || !l_dot->is_target_enum_member()) { // match (someColor) { anotherColor => } + fire(cur_f, v_arm->loc, "wrong pattern matching: `match` should contain members of a enum"); + } + EnumMemberPtr member_ref = std::get(l_dot->target); + if (std::find(covered_enum.begin(), covered_enum.end(), member_ref) != covered_enum.end()) { + fire(cur_f, v_arm->loc, "wrong pattern matching: duplicated enum member in `match`"); + } + covered_enum.push_back(member_ref); + } break; } default: @@ -627,11 +652,19 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } + // only `else` branch + if (has_else_arm && !has_type_arm && !has_expr_arm) { + fire(cur_f, v->loc, "`match` contains only `else`, but no variants"); + } + // fire if `match` by type is not exhaustive - if (has_type_arm && subject_union && subject_union->variants.size() != covered_type_ids.size()) { + if (has_type_arm && subject_union && subject_union->variants.size() != covered_types.size()) { std::string missing; for (TypePtr variant : subject_union->variants) { - if (std::find(covered_type_ids.begin(), covered_type_ids.end(), variant->get_type_id()) == covered_type_ids.end()) { + auto it_mentioned = std::find_if(covered_types.begin(), covered_types.end(), [variant](TypePtr existing) { + return existing->equal_to(variant); + }); + if (it_mentioned == covered_types.end()) { if (!missing.empty()) { missing += ", "; } @@ -640,8 +673,30 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } fire(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); } - // `match` by expression, if it's not statement, should have `else` (unless it's match over bool with const true/false) - if (has_expr_arm && !has_else_arm && !v->is_statement()) { + // fire if `match` by enum is not exhaustive + if (has_expr_arm && subject_enum && !has_else_arm && subject_enum->enum_ref->members.size() != covered_enum.size()) { + std::string missing; + for (EnumMemberPtr member_ref : subject_enum->enum_ref->members) { + if (std::find(covered_enum.begin(), covered_enum.end(), member_ref) == covered_enum.end()) { + if (!missing.empty()) { + missing += ", "; + } + missing += member_ref->name; + } + } + fire(cur_f, v->loc, "`match` does not cover all possible enum members; missing members are: " + missing); + } + // fire if `match` by enum covers all cases, but contains `else` + // (note that `else` for types could exist for a lazy match; for non-lazy, it's fired later) + if (has_expr_arm && subject_enum && has_else_arm && subject_enum->enum_ref->members.size() == covered_enum.size()) { + for (int i = 0; i < v->get_arms_count(); ++i) { + if (auto v_arm = v->get_arm(i); v_arm->pattern_kind == MatchArmKind::else_branch) { + fire(cur_f, v_arm->loc, "`match` already covers all possible enum members, `else` is invalid"); + } + } + } + // `match` by expression, if it's not a statement, should have `else` (unless exhaustive) + if (has_expr_arm && !has_else_arm && !v->is_statement() && !subject_enum) { bool needs_else_branch = true; if (expect_boolean(subject_type) && v->get_arms_count() == 2) { auto arm0 = v->get_arm(0)->get_pattern_expr()->try_as(); @@ -713,7 +768,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { parent::visit(v); - if (!expect_integer(v->get_thrown_code())) { + if (!expect_thrown_code(v->get_thrown_code()->inferred_type)) { fire(cur_f, v->get_thrown_code()->loc, "excNo of `throw` must be an integer, got " + to_string(v->get_thrown_code())); } if (v->has_thrown_arg() && v->get_thrown_arg()->inferred_type->get_width_on_stack() != 1) { @@ -728,7 +783,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (!expect_integer(cond) && !expect_boolean(cond)) { fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); } - if (!expect_integer(v->get_thrown_code())) { + if (!expect_thrown_code(v->get_thrown_code()->inferred_type)) { fire(cur_f, v->get_thrown_code()->loc, "thrown excNo of `assert` must be an integer, got " + to_string(v->get_thrown_code())); } @@ -802,6 +857,16 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire_error_type_mismatch(nullptr, field_ref->loc, "can not assign {src} to {dst}", inferred_type, field_ref->declared_type); } } + + // given enum member `Red = 1` check types within its init_value and the whole expression itself + void start_visiting_enum_member(EnumDefPtr enum_ref, EnumMemberPtr member_ref) { + parent::visit(member_ref->init_value); + + TypePtr inferred_type = member_ref->init_value->inferred_type; + if (!inferred_type->equal_to(TypeDataInt::create()) && !inferred_type->equal_to(TypeDataEnum::create(enum_ref))) { + fire(nullptr, member_ref->loc, "enum member is " + to_string(inferred_type) + ", not `int`\nhint: all enums must be integers"); + } + } }; void pipeline_check_inferred_types() { @@ -818,6 +883,13 @@ void pipeline_check_inferred_types() { } } } + for (EnumDefPtr enum_ref : get_all_declared_enums()) { + for (EnumMemberPtr member_ref : enum_ref->members) { + if (member_ref->has_init_value()) { + visitor.start_visiting_enum_member(enum_ref, member_ref); + } + } + } } } // namespace tolk diff --git a/tolk/pipe-check-private-fields.cpp b/tolk/pipe-check-private-fields.cpp new file mode 100644 index 000000000..46373c7ac --- /dev/null +++ b/tolk/pipe-check-private-fields.cpp @@ -0,0 +1,95 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "platform-utils.h" +#include "type-system.h" + +namespace tolk { + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_private_field_used_outside_method(FunctionPtr cur_f, SrcLocation loc, StructPtr struct_ref, StructFieldPtr field_ref) { + throw ParseError(cur_f, loc, "field `" + struct_ref->as_human_readable() + "." + field_ref->name + "` is private"); +} + +static bool is_private_field_usage_allowed(FunctionPtr cur_f, StructPtr struct_ref) { + // private fields are accessible only inside methods for that struct + if (!cur_f->is_method()) { + return false; + } + const TypeDataStruct* receiver_struct = cur_f->receiver_type->unwrap_alias()->try_as(); + if (receiver_struct && receiver_struct->struct_ref == struct_ref) { + return true; + } + + // probably it's generic, e.g. struct_ref = `Box` and receiver = `Box` + if (struct_ref->is_instantiation_of_generic_struct() && cur_f->is_instantiation_of_generic_function()) { + const auto* receiver_Ts = cur_f->base_fun_ref->receiver_type->try_as(); + return receiver_Ts && receiver_Ts->struct_ref == struct_ref->base_struct_ref; + } + + return false; +} + +class CheckPrivateFieldsUsageVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + +protected: + void visit(V v) override { + parent::visit(v); + + if (v->is_target_struct_field()) { + StructFieldPtr field_ref = std::get(v->target); + const TypeDataStruct* obj_type = v->get_obj()->inferred_type->unwrap_alias()->try_as(); + tolk_assert(obj_type); + if (field_ref->is_private && !is_private_field_usage_allowed(cur_f, obj_type->struct_ref)) { + fire_error_private_field_used_outside_method(cur_f, v->loc, obj_type->struct_ref, field_ref); + } + } + } + + void visit(V v) override { + parent::visit(v); + tolk_assert(v->struct_ref); + + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + StructFieldPtr field_ref = v_field->field_ref; + if (field_ref->is_private && !is_private_field_usage_allowed(cur_f, v->struct_ref)) { + fire_error_private_field_used_outside_method(cur_f, v_field->loc, v->struct_ref, field_ref); + } + } + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + cur_f = nullptr; + } +}; + +void pipeline_check_private_fields_usage() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index cb8067628..69251f348 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -18,6 +18,7 @@ #include "ast.h" #include "ast-visitor.h" #include "platform-utils.h" +#include "type-system.h" /* * This pipe checks lvalue/rvalue for validity. @@ -45,6 +46,11 @@ static void fire_error_modifying_immutable_variable(FunctionPtr cur_f, SrcLocati } } +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_modifying_readonly_field(FunctionPtr cur_f, SrcLocation loc, StructPtr struct_ref, StructFieldPtr field_ref) { + throw ParseError(cur_f, loc, "modifying readonly field `" + struct_ref->as_human_readable() + "." + field_ref->name + "`"); +} + // validate a function used as rvalue, like `var cb = f` // it's not a generic function (ensured earlier at type inferring) and has some more restrictions static void validate_function_used_as_noncall(FunctionPtr cur_f, AnyExprV v, FunctionPtr fun_ref) { @@ -159,10 +165,20 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { // check for `immutableVal.field = rhs` or any other mutation of an immutable tensor/tuple/object // don't allow cheating like `((immutableVal!)).field = rhs` + // same here: check for `obj.readonlyField = rhs` or any other mutation of a readonly field if (v->is_lvalue) { - AnyExprV leftmost_obj = v->get_obj(); + AnyExprV leftmost_obj = v; while (true) { if (auto as_dot = leftmost_obj->try_as()) { + if (as_dot->is_target_struct_field()) { + StructFieldPtr field_ref = std::get(as_dot->target); + const TypeDataStruct* obj_type = as_dot->get_obj()->inferred_type->unwrap_alias()->try_as(); + tolk_assert(obj_type); + if (field_ref->is_readonly) { + fire_error_modifying_readonly_field(cur_f, as_dot->loc, obj_type->struct_ref, field_ref); + } + } + leftmost_obj = as_dot->get_obj(); } else if (auto as_par = leftmost_obj->try_as()) { leftmost_obj = as_par->get_expr(); @@ -179,6 +195,12 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { if (LocalVarPtr var_ref = as_ref->sym->try_as()) { on_var_used_as_lvalue(leftmost_obj->loc, var_ref); } + if (as_ref->sym->try_as()) { // `Point.create = f` + if (v->is_target_enum_member()) { + fire(cur_f, v->loc, "modifying immutable constant"); + } + fire(cur_f, v->loc, "invalid left side of assignment"); + } } } @@ -186,6 +208,8 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { if (v->is_rvalue && v->is_target_fun_ref()) { validate_function_used_as_noncall(cur_f, v, std::get(v->target)); } + + parent::visit(v); } void visit(V v) override { diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index f8b9b9cd0..792a73d29 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -18,6 +18,7 @@ #include "ast.h" #include "ast-visitor.h" #include "pack-unpack-api.h" +#include "maps-kv-api.h" #include "generics-helpers.h" #include "type-system.h" @@ -42,6 +43,43 @@ static void fire_error_theoretical_overflow_1023(StructPtr struct_ref, PackSize ); } +GNU_ATTRIBUTE_NOINLINE +static void check_map_TKey_TValue(SrcLocation loc, TypePtr TKey, TypePtr TValue) { + std::string because_msg; + if (!check_mapKV_TKey_is_valid(TKey, because_msg)) { + fire(nullptr, loc, "invalid `map`: type `" + TKey->as_human_readable() + "` can not be used as a key\n" + because_msg); + } + if (!check_mapKV_TValue_is_valid(TValue, because_msg)) { + fire(nullptr, loc, "invalid `map`: type `" + TValue->as_human_readable() + "` can not be used as a value\n" + because_msg); + } +} + +GNU_ATTRIBUTE_NOINLINE +static void check_mapKV_inside_type(SrcLocation loc, TypePtr any_type) { + any_type->replace_children_custom([loc](TypePtr child) { + if (const TypeDataMapKV* t_map = child->try_as()) { + check_map_TKey_TValue(loc, t_map->TKey, t_map->TValue); + } + if (const TypeDataAlias* t_alias = child->try_as()) { + check_mapKV_inside_type(loc, t_alias->underlying_type); + } + return child; + }); +} + +static void check_mapKV_inside_type(AnyTypeV type_node) { + if (type_node && type_node->resolved_type->has_mapKV_inside()) { + check_mapKV_inside_type(type_node->loc, type_node->resolved_type); + } +} + +// given `enum Role: int8` check colon type (not struct/slice etc.) +static void check_enum_colon_type_to_be_intN(AnyTypeV colon_type_node) { + if (!colon_type_node->resolved_type->try_as()) { + fire(nullptr, colon_type_node->loc, "serialization type of `enum` must be intN: `int8` / `uint32` / etc."); + } +} + class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody { FunctionPtr cur_f = nullptr; @@ -73,12 +111,19 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody } void visit(V v) override { + parent::visit(v); + FunctionPtr fun_ref = v->fun_maybe; - if (!fun_ref || !fun_ref->is_compile_time_special_gen() || !fun_ref->is_instantiation_of_generic_function()) { + if (!fun_ref || !fun_ref->is_builtin_function() || !fun_ref->is_instantiation_of_generic_function()) { return; } - std::string_view f_name = fun_ref->base_fun_ref->name; + + if (f_name == "createEmptyMap") { + check_map_TKey_TValue(v->loc, fun_ref->substitutedTs->typeT_at(0), fun_ref->substitutedTs->typeT_at(1)); + return; + } + TypePtr serialized_type = nullptr; bool is_pack = false; if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || @@ -99,19 +144,42 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody check_type_fits_cell_or_has_policy(serialized_type); } - public: + void visit(V v) override { + check_mapKV_inside_type(v->type_node); + } + +public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } void start_visiting_function(FunctionPtr fun_ref, V v_function) override { cur_f = fun_ref; + + for (int i = 0; i < cur_f->get_num_params(); ++i) { + check_mapKV_inside_type(cur_f->get_param(i).type_node); + } + parent::visit(v_function->get_body()); } }; void pipeline_check_serialized_fields() { visit_ast_of_all_functions(); + + for (StructPtr struct_ref : get_all_declared_structs()) { + for (StructFieldPtr field_ref : struct_ref->fields) { + check_mapKV_inside_type(field_ref->type_node); + } + } + for (GlobalVarPtr glob_ref : get_all_declared_global_vars()) { + check_mapKV_inside_type(glob_ref->type_node); + } + for (EnumDefPtr enum_ref : get_all_declared_enums()) { + if (enum_ref->colon_type_node) { + check_enum_colon_type_to_be_intN(enum_ref->colon_type_node); + } + } } } // namespace tolk diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 74fbb3453..27cecd3c6 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -35,6 +35,21 @@ namespace tolk { +// assign `enum` members values (either auto-compute sequentially or use manual initializers) +static void assign_enum_members_values(EnumDefPtr enum_ref) { + td::RefInt256 prev_value = td::make_refint(-1); + for (EnumMemberPtr member_ref : enum_ref->members) { + td::RefInt256 cur_value = member_ref->init_value ? eval_enum_member_init_value(member_ref->init_value) : prev_value + 1; + if (!cur_value->is_valid() || !cur_value->signed_fits_bits(257)) { + fire(nullptr, member_ref->loc, "integer overflow"); + } + + member_ref->mutate()->assign_computed_value(cur_value); + prev_value = std::move(cur_value); + } +} + + class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { static V create_int_const(SrcLocation loc, td::RefInt256&& intval) { auto v_int = createV(loc, std::move(intval), {}); @@ -190,6 +205,10 @@ void pipeline_constant_folding() { } } } + // assign `enum` members values (either auto-compute sequentially or use manual initializers) + for (EnumDefPtr enum_ref : get_all_declared_enums()) { + assign_enum_members_values(enum_ref); + } replace_ast_of_all_functions(); } diff --git a/tolk/pipe-discover-parse-sources.cpp b/tolk/pipe-discover-parse-sources.cpp index 1cadf07f4..da4a7850f 100644 --- a/tolk/pipe-discover-parse-sources.cpp +++ b/tolk/pipe-discover-parse-sources.cpp @@ -47,7 +47,7 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con tolk_assert(!file->ast); file->ast = parse_src_file_to_ast(file); - // if (!file->is_stdlib_file()) file->ast->debug_print(); + // if (!file->is_stdlib_file) file->ast->debug_print(); for (AnyV v_toplevel : file->ast->as()->get_toplevel_declarations()) { if (auto v_import = v_toplevel->try_as()) { diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index 1dc513f05..8f55737d9 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -85,11 +85,7 @@ static void generate_output_func(FunctionPtr fun_ref) { modifier = "REF"; } if (G.settings.tolk_src_as_line_comments) { - std::cout << " // " << fun_ref->loc; - if (!fun_ref->n_times_called && !fun_ref->is_used_as_noncall() && !fun_ref->is_entrypoint() && !fun_ref->has_tvm_method_id()) { - std::cout << " (note: function never called!)"; - } - std::cout << std::endl; + std::cout << " // " << fun_ref->loc << std::endl; } std::cout << " " << fun_ref->name << "() PROC" << modifier << ":<{"; int mode = 0; @@ -173,7 +169,7 @@ void pipeline_generate_fif_output_to_std_cout() { } for (GlobalVarPtr var_ref : G.all_global_vars) { - if (!var_ref->is_really_used() && G.settings.remove_unused_functions) { + if (!var_ref->is_really_used()) { if (G.is_verbosity(2)) { std::cerr << var_ref->name << ": variable not generated, it's unused\n"; } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 0a0952123..1a80abf34 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -19,6 +19,7 @@ #include "ast.h" #include "ast-visitor.h" #include "generics-helpers.h" +#include "overload-resolution.h" #include "type-system.h" #include "smart-casts-cfg.h" #include @@ -159,14 +160,20 @@ static void fire_error_using_lateinit_variable_uninitialized(FunctionPtr cur_f, // fire an error when `obj.f()`, method `f` not found, try to locate a method for another type GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_method_not_found(FunctionPtr cur_f, SrcLocation loc, TypePtr receiver_type, std::string_view method_name) { - if (std::vector other = lookup_methods_with_name(method_name); !other.empty()) { - fire(cur_f, loc, "method `" + to_string(method_name) + "` not found for type " + to_string(receiver_type) + "\n(but it exists for type " + to_string(other.front()->receiver_type) + ")"); +static void fire_error_method_or_field_not_found(FunctionPtr cur_f, SrcLocation loc, TypePtr receiver_type, std::string_view field_name, bool called_as_method, bool is_static_dot) { + if (!called_as_method && is_static_dot && receiver_type->unwrap_alias()->try_as()) { + fire(cur_f, loc, "member `" + to_string(field_name) + "` does not exist in enum " + to_string(receiver_type)); } - if (const Symbol* sym = lookup_global_symbol(method_name); sym && sym->try_as()) { - fire(cur_f, loc, "method `" + to_string(method_name) + "` not found, but there is a global function named `" + to_string(method_name) + "`\n(a function should be called `foo(arg)`, not `arg.foo()`)"); + if (!called_as_method && !is_static_dot) { + fire(cur_f, loc, "field `" + to_string(field_name) + "` doesn't exist in type " + to_string(receiver_type)); + } + if (std::vector other = lookup_methods_with_name(field_name); !other.empty()) { + fire(cur_f, loc, "method `" + to_string(field_name) + "` not found for type " + to_string(receiver_type) + "\n(but it exists for type " + to_string(other.front()->receiver_type) + ")"); } - fire(cur_f, loc, "method `" + to_string(method_name) + "` not found"); + if (const Symbol* sym = lookup_global_symbol(field_name); sym && sym->try_as()) { + fire(cur_f, loc, "method `" + to_string(field_name) + "` not found, but there is a global function named `" + to_string(field_name) + "`\n(a function should be called `foo(arg)`, not `arg.foo()`)"); + } + fire(cur_f, loc, "method `" + to_string(field_name) + "` not found"); } // safe version of std::stoi that does not crash on long numbers @@ -223,16 +230,10 @@ static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, AliasDefPtr // given `p.create` (called_receiver = Point, called_name = "create") // look up a corresponding method (it may be `Point.create` / `Point?.create` / `T.create`) -static MethodCallCandidate choose_only_method_to_call(FunctionPtr cur_f, SrcLocation loc, TypePtr called_receiver, std::string_view called_name) { - // most practical cases: `builder.storeInt` etc., when a direct method for receiver exists - if (FunctionPtr exact_method = match_exact_method_for_call_not_generic(called_receiver, called_name)) { - return {exact_method, GenericsSubstitutions(exact_method->genericTs)}; - } - - // else, try to match, for example `10.copy` with `int?.copy`, with `T.copy`, etc. - std::vector candidates = match_methods_for_call_including_generic(called_receiver, called_name); +static std::pair choose_only_method_to_call(FunctionPtr cur_f, SrcLocation loc, TypePtr called_receiver, std::string_view called_name) { + std::vector candidates = resolve_methods_for_call(called_receiver, called_name); if (candidates.size() == 1) { - return candidates[0]; + return {candidates[0].method_ref, candidates[0].substitutedTs}; } if (candidates.empty()) { // return nullptr, the caller side decides how to react on this return {nullptr, GenericsSubstitutions(nullptr)}; @@ -240,10 +241,11 @@ static MethodCallCandidate choose_only_method_to_call(FunctionPtr cur_f, SrcLoca std::ostringstream msg; msg << "call to method `" << called_name << "` for type `" << called_receiver << "` is ambiguous\n"; - for (const auto& [method_ref, substitutedTs] : candidates) { + for (const MethodCallCandidate& candidate : candidates) { + FunctionPtr method_ref = candidate.method_ref; msg << "candidate function: `" << method_ref->as_human_readable() << "`"; if (method_ref->is_generic_function()) { - msg << " with " << substitutedTs.as_human_readable(false); + msg << " with " << candidate.substitutedTs.as_human_readable(false); } if (method_ref->loc.is_defined()) { msg << " (declared at " << method_ref->loc << ")\n"; @@ -254,6 +256,12 @@ static MethodCallCandidate choose_only_method_to_call(FunctionPtr cur_f, SrcLoca fire(cur_f, loc, msg.str()); } +static void check_no_unexpected_type_arguments(FunctionPtr cur_f, V v_instantiationTs) { + if (v_instantiationTs != nullptr) { + fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); + } +} + // given fun `f` and a call `f(a,b,c)`, check that argument count is expected; // (parameters may have default values, so it's not as trivial as to compare params and args size) void check_arguments_count_at_fun_call(FunctionPtr cur_f, V v, FunctionPtr called_f, AnyExprV self_obj) { @@ -786,7 +794,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } rhs_type = rhs_type->unwrap_alias(); - TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr expr_type = v->get_expr()->inferred_type; TypePtr non_rhs_type = calculate_type_subtract_rhs_type(expr_type, rhs_type); if (expr_type->equal_to(rhs_type)) { // `expr is ` is always true v->mutate()->assign_always_true_or_false(v->is_negated ? 2 : 1); @@ -823,7 +831,7 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_not_null_operator(V v, FlowContext&& flow, bool used_as_condition) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); TypePtr expr_type = v->get_expr()->inferred_type; - TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type->unwrap_alias(), TypeDataNullLiteral::create()); + TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type, TypeDataNullLiteral::create()); assign_inferred_type(v, without_null_type != TypeDataNever::create() ? without_null_type : expr_type); if (!used_as_condition) { @@ -891,7 +899,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { for (int i = 0; i < substitutedTs.size(); ++i) { - if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1 && !fun_ref->is_variadic_width_T_allowed()) { + if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1 && !is_allowed_asm_generic_function_with_non1_width_T(fun_ref, i)) { fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); } } @@ -907,8 +915,8 @@ class InferTypesAndCallsAndFieldsVisitor final { // - or inside a call: `globalF()` / `genericFn()` / `genericFn()` / `local_var()` if (LocalVarPtr var_ref = v->sym->try_as()) { - TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); - tolk_assert(declared_or_smart_casted != nullptr); // all local vars are presented in flow + tolk_assert(flow.smart_cast_exists(SinkExpression(var_ref))); // all local vars are presented in flow + TypePtr declared_or_smart_casted = flow.smart_cast_or_original(SinkExpression(var_ref), var_ref->declared_type); assign_inferred_type(v, declared_or_smart_casted); if (var_ref->is_lateinit() && declared_or_smart_casted == TypeDataUnknown::create() && v->is_rvalue) { fire_error_using_lateinit_variable_uninitialized(cur_f, v->loc, v->get_name()); @@ -942,9 +950,9 @@ class InferTypesAndCallsAndFieldsVisitor final { fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); v->mutate()->assign_sym(fun_ref); - } else if (UNLIKELY(v_instantiationTs != nullptr)) { + } else { // non-generic function referenced like `return beginCell;` - fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); + check_no_unexpected_type_arguments(cur_f, v_instantiationTs); } if (out_f_called) { // so, it's `globalF()` / `genericFn()` / `genericFn()` @@ -967,9 +975,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } // for non-functions: `local_var` and similar not allowed - if (UNLIKELY(v->has_instantiationTs())) { - fire(cur_f, v->get_instantiationTs()->loc, "type arguments not expected here"); - } + check_no_unexpected_type_arguments(cur_f, v->get_instantiationTs()); return ExprFlow(std::move(flow), used_as_condition); } @@ -982,14 +988,25 @@ class InferTypesAndCallsAndFieldsVisitor final { FunctionPtr fun_ref = nullptr; // to be filled for `.method` / `Point.create` (both standalone or in a call) GenericsSubstitutions substitutedTs(nullptr); V v_ident = v->get_identifier(); // field/method name vertex + V v_instantiationTs = v->get_instantiationTs(); + std::string_view field_name = v_ident->name; - // handle `Point.create` / `Container.wrap`: no dot_obj expression actually, lhs is a type, looking up a method + // handle `Point.create` / `Container.wrap` / `Color.Red`: lhs is a type, looking up a constant/method if (auto obj_ref = v->get_obj()->try_as()) { if (const auto* obj_as_type = obj_ref->sym->try_as()) { TypePtr receiver_type = obj_as_type->resolved_type; - std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, v_ident->loc, receiver_type, v->get_field_name()); + // `Color.Red` (enum member) — just fill v->target and done + if (const TypeDataEnum* enum_receiver = receiver_type->unwrap_alias()->try_as()) { + if (EnumMemberPtr member_ref = enum_receiver->enum_ref->find_member(field_name)) { + v->mutate()->assign_target(member_ref); + check_no_unexpected_type_arguments(cur_f, v_instantiationTs); + assign_inferred_type(v, enum_receiver); // `Color.Red` is `Color` + return ExprFlow(std::move(flow), used_as_condition); + } + } + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, v_ident->loc, receiver_type, field_name); if (!fun_ref) { - fire(cur_f, v_ident->loc, "method `" + to_string(v->get_field_name()) + "` not found for type " + to_string(receiver_type)); + fire_error_method_or_field_not_found(cur_f, v_ident->loc, receiver_type, field_name, out_f_called != nullptr, true); } } } @@ -1001,8 +1018,6 @@ class InferTypesAndCallsAndFieldsVisitor final { // our goal is to fill v->target (field/index/method) knowing type of obj TypePtr obj_type = dot_obj ? dot_obj->inferred_type->unwrap_alias() : TypeDataUnknown::create(); - V v_instantiationTs = v->get_instantiationTs(); - std::string_view field_name = v_ident->name; // check for field access (`user.id`), when obj is a struct if (const TypeDataStruct* obj_struct = obj_type->try_as()) { @@ -1010,10 +1025,9 @@ class InferTypesAndCallsAndFieldsVisitor final { v->mutate()->assign_target(field_ref); TypePtr inferred_type = field_ref->declared_type; if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { - if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { - inferred_type = smart_casted; - } + inferred_type = flow.smart_cast_or_original(s_expr, inferred_type); } + check_no_unexpected_type_arguments(cur_f, v_instantiationTs); assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } @@ -1033,9 +1047,7 @@ class InferTypesAndCallsAndFieldsVisitor final { v->mutate()->assign_target(index_at); TypePtr inferred_type = t_tensor->items[index_at]; if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { - if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { - inferred_type = smart_casted; - } + inferred_type = flow.smart_cast_or_original(s_expr, inferred_type); } assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); @@ -1047,9 +1059,7 @@ class InferTypesAndCallsAndFieldsVisitor final { v->mutate()->assign_target(index_at); TypePtr inferred_type = t_tuple->items[index_at]; if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { - if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { - inferred_type = smart_casted; - } + inferred_type = flow.smart_cast_or_original(s_expr, inferred_type); } assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); @@ -1084,11 +1094,7 @@ class InferTypesAndCallsAndFieldsVisitor final { fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\nhint: check it via `obj != null` or use non-null assertion `obj!` operator"); } } - if (out_f_called) { - fire_error_method_not_found(cur_f, v_ident->loc, dot_obj->inferred_type, v->get_field_name()); - } else { - fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(dot_obj)); - } + fire_error_method_or_field_not_found(cur_f, v_ident->loc, dot_obj->inferred_type, field_name, out_f_called != nullptr, false); } // if `fun T.copy(self)` and reference `int.copy` — all generic parameters are determined by the receiver, we know it @@ -1107,9 +1113,9 @@ class InferTypesAndCallsAndFieldsVisitor final { substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(v->loc, fun_ref->genericTs, v_instantiationTs)); fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); - } else if (UNLIKELY(v_instantiationTs != nullptr)) { + } else { // non-generic method referenced like `var cb = c.hash;` - fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); + check_no_unexpected_type_arguments(cur_f, v_instantiationTs); } if (out_f_called) { // so, it's `user.method()` / `t.tupleAt()` / `t.tupleAt()` / `Point.create` @@ -1185,7 +1191,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (param_type->has_genericT_inside()) { param_type = deducingTs.auto_deduce_from_argument(cur_f, self_obj->loc, param_type, self_obj->inferred_type); } - if (param_0.is_mutate_parameter() && self_obj->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { + if (param_0.is_mutate_parameter() && !self_obj->inferred_type->equal_to(param_type)) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(self_obj)) { assign_inferred_type(self_obj, calc_declared_type_before_smart_cast(self_obj)); flow.register_known_type(s_expr, param_type); @@ -1212,7 +1218,7 @@ class InferTypesAndCallsAndFieldsVisitor final { param_type = deducingTs.auto_deduce_from_argument(cur_f, arg_i->loc, param_type, arg_i->inferred_type); } assign_inferred_type(v->get_arg(i), arg_i); // arg itself is an expression - if (param_i.is_mutate_parameter() && arg_i->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { + if (param_i.is_mutate_parameter() && !arg_i->inferred_type->equal_to(param_type)) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(arg_i)) { assign_inferred_type(arg_i, calc_declared_type_before_smart_cast(arg_i)); flow.register_known_type(s_expr, param_type); @@ -1634,7 +1640,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!body_end.is_unreachable()) { fun_ref->mutate()->assign_is_implicit_return(); - if (fun_ref->declared_return_type == TypeDataNever::create()) { // `never` can only be declared, it can't be inferred + if (fun_ref->declared_return_type == TypeDataNever::create()) { fire(fun_ref, v_function->get_body()->as()->loc_end, "a function returning `never` can not have a reachable endpoint"); } } @@ -1654,7 +1660,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } inferred_return_type = return_unifier.get_result(); if (inferred_return_type == nullptr) { // if no return statements at all - inferred_return_type = TypeDataVoid::create(); + inferred_return_type = body_end.is_unreachable() ? TypeDataNever::create() : TypeDataVoid::create(); } if (!body_end.is_unreachable() && inferred_return_type != TypeDataVoid::create()) { @@ -1707,6 +1713,11 @@ class InferTypesAndCallsAndFieldsVisitor final { } } } + + // given `enum Color { Red = 1 }` infer that it's int + void start_visiting_enum_member(EnumMemberPtr member_ref) { + infer_any_expr(member_ref->init_value, FlowContext(), false); + } }; class LaunchInferTypesAndMethodsOnce final { @@ -1769,8 +1780,16 @@ static void infer_and_save_type_of_constant(GlobalConstPtr const_ref) { } void pipeline_infer_types_and_calls_and_fields() { + // loop over user-defined functions visit_ast_of_all_functions(); + // assign inferred_type to built-in functions like __throw() + for (FunctionPtr fun_ref : get_all_builtin_functions()) { + if (LaunchInferTypesAndMethodsOnce::should_visit_function(fun_ref)) { + infer_and_save_return_type_of_function(fun_ref); + } + } + // analyze constants that weren't referenced by any function InferTypesAndCallsAndFieldsVisitor visitor; for (GlobalConstPtr const_ref : get_all_declared_constants()) { @@ -1785,6 +1804,15 @@ void pipeline_infer_types_and_calls_and_fields() { visitor.start_visiting_struct_fields(struct_ref); } } + + // infer types for init_value of enum members + for (EnumDefPtr enum_ref : get_all_declared_enums()) { + for (EnumMemberPtr member_ref : enum_ref->members) { + if (member_ref->has_init_value()) { + visitor.start_visiting_enum_member(member_ref); + } + } + } } void pipeline_infer_types_and_calls_and_fields(FunctionPtr fun_ref) { diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp index 64438645b..8eb0d76d0 100644 --- a/tolk/pipe-lazy-load-insertions.cpp +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -489,7 +489,7 @@ struct ExprUsagesWhileCollecting { ith_field_action.reserve(future_fields.size()); for (int field_idx = 0; field_idx < static_cast(future_fields.size()); ++field_idx) { FutureField f = future_fields[field_idx]; - StructFieldPtr created = new StructFieldData(static_cast(f.field_name), {}, field_idx, nullptr, nullptr); + StructFieldPtr created = new StructFieldData(static_cast(f.field_name), {}, field_idx, false, false, nullptr, nullptr); created->mutate()->assign_resolved_type(f.field_type); hidden_fields.push_back(created); ith_field_action.push_back(f.action); diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 917722946..7f684259a 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -98,7 +98,7 @@ static void validate_arg_ret_order_of_asm_function(V v_body, int n static GlobalConstPtr register_constant(V v) { GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->type_node, v->get_init_value()); - G.symtable.add_global_const(c_sym); + G.symtable.add_global_symbol(c_sym); G.all_constants.push_back(c_sym); v->mutate()->assign_const_ref(c_sym); return c_sym; @@ -107,7 +107,7 @@ static GlobalConstPtr register_constant(V v) { static GlobalVarPtr register_global_var(V v) { GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->type_node); - G.symtable.add_global_var(g_sym); + G.symtable.add_global_symbol(g_sym); G.all_global_vars.push_back(g_sym); v->mutate()->assign_glob_ref(g_sym); return g_sym; @@ -122,11 +122,36 @@ static AliasDefPtr register_type_alias(V v, AliasDef AliasDefData* a_sym = new AliasDefData(std::move(name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); a_sym->base_alias_ref = base_alias_ref; // for `Response`, here is `Response` - G.symtable.add_type_alias(a_sym); + G.symtable.add_global_symbol(a_sym); v->mutate()->assign_alias_ref(a_sym); return a_sym; } +static EnumDefPtr register_enum(V v) { + auto v_body = v->get_enum_body(); + + std::vector members; + members.reserve(v_body->get_num_members()); + for (int i = 0; i < v_body->get_num_members(); ++i) { + auto v_member = v_body->get_member(i); + std::string member_name = static_cast(v_member->get_identifier()->name); + + for (EnumMemberPtr prev : members) { + if (UNLIKELY(prev->name == member_name)) { + v_member->error("redeclaration of member `" + member_name + "`"); + } + } + members.emplace_back(new EnumMemberData(member_name, v_member->loc, v_member->init_value)); + } + + EnumDefData* e_sym = new EnumDefData(static_cast(v->get_identifier()->name), v->loc, v->colon_type, std::move(members)); + + G.symtable.add_global_symbol(e_sym); + G.all_enums.push_back(e_sym); + v->mutate()->assign_enum_ref(e_sym); + return e_sym; +} + static StructPtr register_struct(V v, StructPtr base_struct_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { auto v_body = v->get_struct_body(); @@ -141,7 +166,7 @@ static StructPtr register_struct(V v, StructPtr base_str v_field->error("redeclaration of field `" + field_name + "`"); } } - fields.emplace_back(new StructFieldData(field_name, v_field->loc, i, v_field->type_node, v_field->default_value)); + fields.emplace_back(new StructFieldData(field_name, v_field->loc, i, v_field->is_private, v_field->is_readonly, v_field->type_node, v_field->default_value)); } PackOpcode opcode(0, 0); @@ -170,7 +195,7 @@ static StructPtr register_struct(V v, StructPtr base_str StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), opcode, v->overflow1023_policy, genericTs, substitutedTs, v); s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` - G.symtable.add_struct(s_sym); + G.symtable.add_global_symbol(s_sym); G.all_structs.push_back(s_sym); v->mutate()->assign_struct_ref(s_sym); return s_sym; @@ -284,6 +309,9 @@ static void iterate_through_file_symbols(const SrcFile* file) { case ast_type_alias_declaration: register_type_alias(v->as()); break; + case ast_enum_declaration: + register_enum(v->as()); + break; case ast_struct_declaration: register_struct(v->as()); break; diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 46710cb63..320ebf70c 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -64,6 +64,15 @@ static void fire_error_type_used_as_symbol(FunctionPtr cur_f, V } } +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_using_self_not_in_method(FunctionPtr cur_f, SrcLocation loc) { + if (cur_f->is_static_method()) { + fire(cur_f, loc, "using `self` in a static method"); + } else { + fire(cur_f, loc, "using `self` in a regular function (not a method)"); + } +} + struct NameAndScopeResolver { std::vector> scopes; @@ -157,7 +166,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (!sym) { fire_error_undefined_symbol(cur_f, v->get_identifier()); } - if (sym->try_as() || sym->try_as()) { + if (sym->try_as() || sym->try_as() || sym->try_as()) { fire_error_type_used_as_symbol(cur_f, v->get_identifier()); } v->mutate()->assign_sym(sym); @@ -174,9 +183,12 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { try { parent::visit(v->get_obj()); } catch (const ParseError&) { - if (v->get_obj()->kind == ast_reference) { - // for `Point.create` / `int.zero`, "undefined symbol" is fired for Point/int + if (auto v_type_name = v->get_obj()->try_as()) { + // for `Point.create` / `int.zero` / `Color.Red`, "undefined symbol" is fired for Point/int/Color // suppress this exception till a later pipe, it will be tried to be resolved as a type + if (v_type_name->get_identifier()->name == "self") { + fire_error_using_self_not_in_method(cur_f, v_type_name->loc); + } return; } throw; @@ -286,6 +298,15 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } } } + + void start_visiting_enum_members(EnumDefPtr enum_ref) { + // member `Red = Another.Blue`, resolve `Another` + for (EnumMemberPtr member_ref : enum_ref->members) { + if (member_ref->has_init_value()) { + parent::visit(member_ref->init_value); + } + } + } }; NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; @@ -308,6 +329,10 @@ void pipeline_resolve_identifiers_and_assign_symbols() { } else if (auto v_struct = v->try_as()) { tolk_assert(v_struct->struct_ref); visitor.start_visiting_struct_fields(v_struct->struct_ref); + + } else if (auto v_enum = v->try_as()) { + tolk_assert(v_enum->enum_ref); + visitor.start_visiting_enum_members(v_enum->enum_ref); } } } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 7ca035cfd..8913403b1 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -170,6 +170,12 @@ class TypeNodesVisitorResolver { // if we're inside `f`, replace "T" with TypeDataInt return substitutedTs->get_substitution_for_nameT(text); } + if (text == "map") { + if (!allow_without_type_arguments) { + fire_error_generic_type_used_without_T(cur_f, loc, "map"); + } + return TypeDataMapKV::create(TypeDataGenericT::create("K"), TypeDataGenericT::create("V")); + } if (const Symbol* sym = lookup_global_symbol(text)) { if (TypePtr custom_type = try_resolve_user_defined_type(cur_f, loc, sym, allow_without_type_arguments)) { if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) { @@ -265,6 +271,9 @@ class TypeNodesVisitorResolver { } return TypeDataStruct::create(struct_ref); } + if (EnumDefPtr enum_ref = sym->try_as()) { + return TypeDataEnum::create(enum_ref); + } return nullptr; } @@ -298,6 +307,12 @@ class TypeNodesVisitorResolver { } return TypeDataAlias::create(instantiate_generic_alias(alias_ref, GenericsSubstitutions(alias_ref->genericTs, type_arguments))); } + if (const TypeDataMapKV* t_map = type_to_instantiate->try_as(); t_map && t_map->TKey->try_as()) { + if (type_arguments.size() != 2) { + fire(cur_f, loc, "type `map` expects 2 type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + } + return TypeDataMapKV::create(type_arguments[0], type_arguments[1]); + } if (const TypeDataGenericT* asT = type_to_instantiate->try_as()) { fire_error_unknown_type_name(cur_f, loc, asT->nameT); } @@ -580,6 +595,23 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } } } + + void start_visiting_enum_members(EnumDefPtr enum_ref) { + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr, false); + + // same for struct field `v: int8 = 0 as int8` + for (EnumMemberPtr member_ref : enum_ref->members) { + if (member_ref->has_init_value()) { + parent::visit(member_ref->init_value); + } + } + + // serialization type: `enum Role: int8` + if (enum_ref->colon_type_node) { + TypePtr colon_type = finalize_type_node(enum_ref->colon_type_node); + enum_ref->mutate()->assign_resolved_colon_type(colon_type); // later it will be checked to be intN + } + } }; // prevent recursion like `struct A { field: A }`; @@ -652,6 +684,9 @@ void pipeline_resolve_types_and_aliases() { TypeNodesVisitorResolver::visit_symbol(v_struct->struct_ref); } visitor.start_visiting_struct_fields(v_struct->struct_ref); + + } else if (auto v_enum = v->try_as()) { + visitor.start_visiting_enum_members(v_enum->enum_ref); } } } diff --git a/tolk/pipeline.h b/tolk/pipeline.h index 641beb06b..d433afcfc 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -40,6 +40,7 @@ void pipeline_infer_types_and_calls_and_fields(); void pipeline_check_inferred_types(); void pipeline_refine_lvalue_for_mutate_arguments(); void pipeline_check_rvalue_lvalue(); +void pipeline_check_private_fields_usage(); void pipeline_check_pure_impure_operations(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index 2b4d80be8..b5ef61aa7 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -14,8 +14,8 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "send-message-api.h" #include "pack-unpack-serializers.h" +#include "generics-helpers.h" #include "type-system.h" namespace tolk { @@ -32,17 +32,6 @@ static bool is_type_UnsafeBodyNoRef_T(TypePtr bodyT) { return false; } -// currently, there is no way to pass custom pack options to createMessage, using hardcoded ones -static std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc) { - StructPtr s_PackOptions = lookup_global_symbol("PackOptions")->try_as(); - std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_PackOptions), loc, "(pack-options)"); - tolk_assert(ir_options.size() == 1); - - var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); - code.emplace_back(loc, Op::_Let, std::vector{ir_options[0]}, std::vector{ir_zero}); // skipBitsNFieldsValidation - return ir_options; -} - // calculate `addrHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` static void append_bitwise_and_shard_mask(CodeBlob& code, SrcLocation loc, var_idx_t ir_addr_hash, var_idx_t ir_shard_depth) { var_idx_t ir_one = code.create_int(loc, 1, "(one)"); @@ -55,38 +44,50 @@ static void append_bitwise_and_shard_mask(CodeBlob& code, SrcLocation loc, var_i // struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } struct IR_AutoDeployAddress { - std::vector is_ContractState, // stateInit is ContractState - is_AddressSharding; // toShard is not null + std::vector ir_stateInitField, + ir_toShardField; + const TypeDataUnion* t_stateInit; + const TypeDataUnion* t_toShard; var_idx_t workchain, // workchain stateInitCode, stateInitData, stateInitCell, // stateInit ir_shardDepth, ir_closeTo; // toShard IR_AutoDeployAddress(CodeBlob& code, SrcLocation loc, const std::vector& ir_vars) { StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); - const TypeDataUnion* t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); - const TypeDataUnion* t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); + t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); + t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); tolk_assert(ir_vars.size() == 1 + 3 + 3); tolk_assert(t_stateInit && t_stateInit->get_width_on_stack() == (2+1) && t_stateInit->size() == 2); tolk_assert(t_toShard && t_toShard->get_width_on_stack() == (2+1) && t_toShard->or_null); workchain = ir_vars[0]; - std::vector ir_stateInitUnion(ir_vars.begin() + 1, ir_vars.begin() + 1 + 3); - is_ContractState = pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_stateInitUnion, loc, "(is-ContractState)"); - std::vector ir_ContractState = transition_to_target_type(std::vector(ir_stateInitUnion), code, t_stateInit, t_stateInit->variants[0], loc); + ir_stateInitField = std::vector(ir_vars.begin() + 1, ir_vars.begin() + 1 + 3); + std::vector ir_ContractState = transition_to_target_type(std::vector(ir_stateInitField), code, t_stateInit, t_stateInit->variants[0], loc); stateInitCode = ir_ContractState[0]; stateInitData = ir_ContractState[1]; - stateInitCell = transition_to_target_type(std::vector(ir_stateInitUnion), code, t_stateInit, t_stateInit->variants[1], loc)[0]; + stateInitCell = transition_to_target_type(std::vector(ir_stateInitField), code, t_stateInit, t_stateInit->variants[1], loc)[0]; - std::vector ir_toShardOrNull(ir_vars.begin() + 1 + 3, ir_vars.begin() + 1 + 3 + 3); - is_AddressSharding = pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_toShardOrNull, loc, "(is-AddressSharding)"); - std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_toShardOrNull), code, t_toShard, t_toShard->or_null, loc); + ir_toShardField = std::vector(ir_vars.begin() + 1 + 3, ir_vars.begin() + 1 + 3 + 3); + std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_toShardField), code, t_toShard, t_toShard->or_null, loc); ir_shardDepth = ir_AddressSharding[0]; ir_closeTo = ir_AddressSharding[1]; } + + // generate IR vars "stateInit is ContractState" + std::vector is_ContractState(CodeBlob& code, SrcLocation loc) const { + return pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_stateInitField, loc, "(is-ContractState)"); + } + + // generate IR vars "toShard is not null" + std::vector is_AddressSharding(CodeBlob& code, SrcLocation loc) const { + return pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_toShardField, loc, "(is-AddressSharding)"); + } }; -std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { +// fun createMessage(options: CreateMessageOptions): OutMessage +std::vector generate_createMessage(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_options) { + TypePtr bodyT = called_f->substitutedTs->typeT_at(0); StructPtr s_Options = lookup_global_symbol("CreateMessageOptions")->try_as(); StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); @@ -98,6 +99,7 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T tolk_assert(t_value && t_value->get_width_on_stack() == (2+1) && t_value->size() == 2); int offset = 0; + std::vector rvect = ir_options[0]; auto next_slice = [&rvect, &offset](int width) -> std::vector { int start = offset; offset += width; @@ -115,14 +117,11 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T // struct ContractState { code: cell; data: cell; } // struct AddressShardingOptions { fixedPrefixLength: uint5; closeTo: address; } std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, loc, "(is-address)"); - std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, loc, "(is-address)"); + std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, loc, "(is-auto)"); std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); std::vector ir_dest_AutoDeployAddress = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), loc); IR_AutoDeployAddress ir_dest_ad(code, loc, ir_dest_AutoDeployAddress); - // currently, there is no way to pass PackOptions, defaults are used - std::vector ir_options = create_default_PackOptions(code, loc); - FunctionPtr f_beginCell = lookup_function("beginCell"); FunctionPtr f_endCell = lookup_function("builder.endCell"); @@ -143,7 +142,7 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T if (body_store_as_ref && !body_already_ref) { std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), loc, "refb"); code.emplace_back(loc, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); - PackContext ref_ctx(code, loc, ir_ref_builder, ir_options); + PackContext ref_ctx(code, loc, ir_ref_builder, create_default_PackOptions(code, loc)); ref_ctx.generate_pack_any(bodyT, std::move(ir_body)); std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(ref-cell)"); code.emplace_back(loc, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); @@ -152,7 +151,7 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T std::vector ir_builder = code.create_var(TypeDataSlice::create(), loc, "b"); code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); - PackContext ctx(code, loc, ir_builder, ir_options); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); var_idx_t ir_one = code.create_int(loc, 1, "(one)"); @@ -185,11 +184,11 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast ctx.storeInt(ir_dest_ad.workchain, 8); std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState(code, loc)); { // input is `dest: { ... stateInit: { code, data } }` code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding(code, loc)); { // input is `dest: { ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } }; // then stateInitHash = (hash of StateInit = 0b1(depth)0110 (prefix + code + data)) @@ -215,7 +214,7 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); code.close_pop_cur(loc); } - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding(code, loc)); { // input is `dest: { ... toShard: { fixedPrefixLength, closeTo } }` // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; @@ -303,12 +302,12 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T } { code.push_set_cur(if_no_init.block0); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState(code, loc)); { // input is `dest: { ... stateInit: { code, data } }` and need to compose TL/B StateInit; // it's either just code+data OR (if `toShard: { ... }` is set) fixedPrefixLength+code+data code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding(code, loc)); { // 1 (maybe true) + 0 (either left) + 1 (maybe true of StateInit) + fixedPrefixLength + 0110 + body ref or not code.push_set_cur(if_sharded.block0); @@ -356,7 +355,9 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T return ir_cell; } -std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { +// fun createExternalLogMessage(options: CreateExternalLogMessageOptions): OutMessage +std::vector generate_createExternalLogMessage(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + TypePtr bodyT = called_f->substitutedTs->typeT_at(0); StructPtr s_Options = lookup_global_symbol("CreateExternalLogMessageOptions")->try_as(); StructPtr s_ExtOutLogBucket = lookup_global_symbol("ExtOutLogBucket")->try_as(); @@ -366,6 +367,7 @@ std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLoca tolk_assert(t_topic && t_topic->get_width_on_stack() == (1+1) && t_topic->size() == 2); int offset = 0; + std::vector rvect = args[0]; auto next_slice = [&rvect, &offset](int width) -> std::vector { int start = offset; offset += width; @@ -490,7 +492,9 @@ std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLoca return ir_cell; } -std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options) { +// fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions): builder +std::vector generate_address_buildInAnotherShard(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& args) { + std::vector ir_shard_options = args[1]; tolk_assert(ir_shard_options.size() == 2); // example for fixedPrefixLength (shard depth) = 8: @@ -515,14 +519,15 @@ std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcL std::vector ir_restLenA = {code.create_int(loc, 256, "(last-addrA)")}; code.emplace_back(loc, Op::_Call, ir_restLenA, std::vector{ir_restLenA[0], ir_shard_options[0]}, lookup_function("_-_")); std::vector ir_tailA = code.create_tmp_var(TypeDataSlice::create(), loc, "(tailA)"); - code.emplace_back(loc, Op::_Call, ir_tailA, std::vector{ir_self_address[0], ir_restLenA[0]}, lookup_function("slice.getLastBits")); + code.emplace_back(loc, Op::_Call, ir_tailA, std::vector{args[0][0], ir_restLenA[0]}, lookup_function("slice.getLastBits")); code.emplace_back(loc, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_tailA[0]}, lookup_function("builder.storeSlice")); return ir_builder; } -std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy) { - IR_AutoDeployAddress ir_self(code, loc, ir_auto_deploy); +// fun AutoDeployAddress.buildAddress(self): builder +std::vector generate_AutoDeployAddress_buildAddress(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_options) { + IR_AutoDeployAddress ir_self(code, loc, ir_options[0]); std::vector ir_builder = code.create_tmp_var(TypeDataSlice::create(), loc, "(addr-b)"); // important! unlike `createMessage()`, we calculate hash and shard prefix BEFORE creating a cell @@ -530,11 +535,11 @@ std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, S // calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState(code, loc)); { // called `{ ... stateInit: { code, data } }` code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); { // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } code.push_set_cur(if_sharded.block0); @@ -560,7 +565,7 @@ std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, S } // now, if toShard, perform bitwise calculations with hashes (order on a stack matters) - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); { // called `{ ... toShard: { fixedPrefixLength, closeTo } }` // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; @@ -599,16 +604,17 @@ std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, S return ir_builder; } -std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy, std::vector&& ir_address) { - IR_AutoDeployAddress ir_self(code, loc, ir_auto_deploy); +// fun AutoDeployAddress.addressMatches(self, addr: address): bool +std::vector generate_AutoDeployAddress_addressMatches(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector>& ir_self_and_addr) { + IR_AutoDeployAddress ir_self(code, loc, ir_self_and_addr[0]); // at first, calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState(code, loc)); { // called `{ ... stateInit: { code, data } }` code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); { // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } code.push_set_cur(if_sharded.block0); @@ -634,7 +640,7 @@ std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, } // now calculate `stateInitHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` - Op& if_sharded1 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + Op& if_sharded1 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); { code.push_set_cur(if_sharded1.block0); append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_self.ir_shardDepth); @@ -647,10 +653,10 @@ std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, // now do `(wc, hash) = addr.getWorkchainAndHash()` std::vector ir_addr_wc_hash = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(self-wc-hash)"); - code.emplace_back(loc, Op::_Call, ir_addr_wc_hash, ir_address, lookup_function("address.getWorkchainAndHash")); + code.emplace_back(loc, Op::_Call, ir_addr_wc_hash, ir_self_and_addr[1], lookup_function("address.getWorkchainAndHash")); // now calculate `hash &= mask` (the same as we did earlier for stateInitHash) - Op& if_sharded2 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + Op& if_sharded2 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding(code, loc)); { code.push_set_cur(if_sharded2.block0); append_bitwise_and_shard_mask(code, loc, ir_addr_wc_hash[1], ir_self.ir_shardDepth); diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 7dd60ee97..1bcbe546b 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -204,13 +204,6 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b, bool* became_union = nul return TypeDataBrackets::create(std::move(types_lca)); } - if (const auto* a_alias = a->try_as()) { - return calculate_type_lca(a_alias->underlying_type, b, became_union); - } - if (const auto* b_alias = b->try_as()) { - return calculate_type_lca(a, b_alias->underlying_type, became_union); - } - TypePtr resulting_union = TypeDataUnion::create(std::vector{a, b}); if (became_union != nullptr && !a->equal_to(resulting_union) && !b->equal_to(resulting_union)) { *became_union = true; @@ -314,6 +307,22 @@ void FlowContext::invalidate_all_subfields(LocalVarPtr var_ref, uint64_t parent_ } } +// get the resulting type of variable or struct field +TypePtr FlowContext::smart_cast_or_original(SinkExpression s_expr, TypePtr originally_declared_type) const { + auto it = known_facts.find(s_expr); + if (it == known_facts.end()) { + return originally_declared_type; + } + + TypePtr smart_casted = it->second.expr_type; + if (smart_casted->equal_to(originally_declared_type)) { + // given `var a: dict`, after merging control flow branches, restore `a: dict` instead of `a: cell?` + // (same for struct fields and other sink expressions) + return originally_declared_type; + } + return smart_casted; +} + // update current type of `local_var` / `tensorVar.0` / `obj.field` // example: `local_var = rhs` // example: `f(mutate obj.field)` @@ -386,26 +395,26 @@ FlowContext FlowContext::merge_flow(FlowContext&& c1, FlowContext&& c2) { // example: `int | slice | builder | bool` - `bool | slice` = `int | builder` // what for: `if (x != null)` / `if (x is T)`, to smart cast x inside if TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type) { - const TypeDataUnion* lhs_union = type->try_as(); + const TypeDataUnion* lhs_union = type->unwrap_alias()->try_as(); if (!lhs_union) { return TypeDataNever::create(); } std::vector rest_variants; - if (const TypeDataUnion* sub_union = subtract_type->try_as()) { + if (const TypeDataUnion* sub_union = subtract_type->unwrap_alias()->try_as()) { if (lhs_union->has_all_variants_of(sub_union)) { rest_variants.reserve(lhs_union->size() - sub_union->size()); for (TypePtr lhs_variant : lhs_union->variants) { - if (!sub_union->has_variant_with_type_id(lhs_variant)) { + if (!sub_union->has_variant_equal_to(lhs_variant)) { rest_variants.push_back(lhs_variant); } } } - } else if (lhs_union->has_variant_with_type_id(subtract_type)) { + } else if (lhs_union->has_variant_equal_to(subtract_type)) { rest_variants.reserve(lhs_union->size() - 1); for (TypePtr lhs_variant : lhs_union->variants) { - if (lhs_variant->get_type_id() != subtract_type->get_type_id()) { + if (!lhs_variant->equal_to(subtract_type)) { rest_variants.push_back(lhs_variant); } } @@ -519,14 +528,10 @@ TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rh // example: `var x: int | slice | cell = 4`, result is int // example: `var x: T1 | T2 | T3 = y as T3 | T1`, result is `T1 | T3` if (const TypeDataUnion* rhs_union = rhs_inferred_type->try_as()) { - bool lhs_has_all_variants_of_rhs = true; - for (TypePtr rhs_variant : rhs_union->variants) { - lhs_has_all_variants_of_rhs &= lhs_union->has_variant_with_type_id(rhs_variant); - } - if (lhs_has_all_variants_of_rhs && rhs_union->size() < lhs_union->size()) { + if (lhs_union->has_all_variants_of(rhs_union) && rhs_union->size() < lhs_union->size()) { std::vector subtypes_of_lhs; for (TypePtr lhs_variant : lhs_union->variants) { - if (rhs_union->has_variant_with_type_id(lhs_variant)) { + if (rhs_union->has_variant_equal_to(lhs_variant)) { subtypes_of_lhs.push_back(lhs_variant); } } diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h index b579e8f31..1fb491d8f 100644 --- a/tolk/smart-casts-cfg.h +++ b/tolk/smart-casts-cfg.h @@ -158,10 +158,8 @@ class FlowContext { bool is_unreachable() const { return unreachable; } - TypePtr smart_cast_if_exists(SinkExpression s_expr) const { - auto it = known_facts.find(s_expr); - return it == known_facts.end() ? nullptr : it->second.expr_type; - } + bool smart_cast_exists(SinkExpression s_expr) const { return known_facts.find(s_expr) != known_facts.end(); } + TypePtr smart_cast_or_original(SinkExpression s_expr, TypePtr originally_declared_type) const; void register_known_type(SinkExpression s_expr, TypePtr assigned_type); void mark_unreachable(UnreachableKind reason); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 348fac839..20b9617b4 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -55,6 +55,10 @@ std::string StructData::as_human_readable() const { return name + genericTs->as_human_readable(); } +std::string EnumDefData::as_human_readable() const { + return name; +} + LocalVarPtr FunctionData::find_param(std::string_view name) const { for (const LocalVarData& param_data : parameters) { if (param_data.name == name) { @@ -66,7 +70,7 @@ LocalVarPtr FunctionData::find_param(std::string_view name) const { bool FunctionData::does_need_codegen() const { // when a function is declared, but not referenced from code in any way, don't generate its body - if (!is_really_used() && G.settings.remove_unused_functions) { + if (!is_really_used()) { return false; } // functions with asm body don't need code generation @@ -189,6 +193,14 @@ void AliasDefData::assign_resolved_type(TypePtr underlying_type) { this->underlying_type = underlying_type; } +void EnumMemberData::assign_init_value(AnyExprV init_value) { + this->init_value = init_value; +} + +void EnumMemberData::assign_computed_value(td::RefInt256 computed_value) { + this->computed_value = std::move(computed_value); +} + void StructFieldData::assign_resolved_type(TypePtr declared_type) { this->declared_type = declared_type; } @@ -203,6 +215,10 @@ void StructData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) } } +void EnumDefData::assign_resolved_colon_type(TypePtr colon_type) { + this->colon_type = colon_type; +} + StructFieldPtr StructData::find_field(std::string_view field_name) const { for (StructFieldPtr field : fields) { if (field->name == field_name) { @@ -230,6 +246,15 @@ std::string StructData::PackOpcode::format_as_slice() const { return result; } +EnumMemberPtr EnumDefData::find_member(std::string_view member_name) const { + for (EnumMemberPtr member_ref : members) { + if (member_ref->name == member_name) { + return member_ref; + } + } + return nullptr; +} + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { SrcLocation prev_loc = previous->loc; @@ -242,52 +267,14 @@ static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* pre throw ParseError(loc, "redefinition of built-in symbol"); } -void GlobalSymbolTable::add_function(FunctionPtr f_sym) { - auto key = key_hash(f_sym->name); - auto [it, inserted] = entries.emplace(key, f_sym); - if (!inserted) { - fire_error_redefinition_of_symbol(f_sym->loc, it->second); - } -} - -void GlobalSymbolTable::add_global_var(GlobalVarPtr g_sym) { - auto key = key_hash(g_sym->name); - auto [it, inserted] = entries.emplace(key, g_sym); - if (!inserted) { - fire_error_redefinition_of_symbol(g_sym->loc, it->second); - } -} - -void GlobalSymbolTable::add_global_const(GlobalConstPtr c_sym) { - auto key = key_hash(c_sym->name); - auto [it, inserted] = entries.emplace(key, c_sym); - if (!inserted) { - fire_error_redefinition_of_symbol(c_sym->loc, it->second); - } -} - -void GlobalSymbolTable::add_type_alias(AliasDefPtr a_sym) { - auto key = key_hash(a_sym->name); - auto [it, inserted] = entries.emplace(key, a_sym); - if (!inserted) { - fire_error_redefinition_of_symbol(a_sym->loc, it->second); - } -} - -void GlobalSymbolTable::add_struct(StructPtr s_sym) { - auto key = key_hash(s_sym->name); - auto [it, inserted] = entries.emplace(key, s_sym); +void GlobalSymbolTable::add_global_symbol(const Symbol* sym) { + auto key = key_hash(sym->name); + auto [it, inserted] = entries.emplace(key, sym); if (!inserted) { - fire_error_redefinition_of_symbol(s_sym->loc, it->second); + fire_error_redefinition_of_symbol(sym->loc, it->second); } } -void GlobalSymbolTable::replace_function(FunctionPtr f_sym) { - auto key = key_hash(f_sym->name); - assert(entries.contains(key)); - entries[key] = f_sym; -} - const Symbol* lookup_global_symbol(std::string_view name) { return G.symtable.lookup(name); } diff --git a/tolk/symtable.h b/tolk/symtable.h index 268b474d3..09235eb35 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -96,13 +96,15 @@ struct LocalVarData final : Symbol { struct FunctionBodyCode; struct FunctionBodyAsm; -struct FunctionBodyBuiltin; +struct FunctionBodyBuiltinAsmOp; +struct FunctionBodyBuiltinGenerateOps; struct GenericsDeclaration; typedef std::variant< FunctionBodyCode*, FunctionBodyAsm*, - FunctionBodyBuiltin* + FunctionBodyBuiltinAsmOp*, + FunctionBodyBuiltinGenerateOps* > FunctionBody; struct FunctionData final : Symbol { @@ -120,7 +122,6 @@ struct FunctionData final : Symbol { flagReturnsSelf = 1024, // return type is `self` (returns the mutated 1st argument), calls can be chainable flagReallyUsed = 2048, // calculated via dfs from used functions; declared but unused functions are not codegenerated flagCompileTimeVal = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others - flagCompileTimeGen = 8192, // at compile-time it's handled specially, not as a regular function: `T.toCell`, etc. flagAllowAnyWidthT = 16384, // for built-in generic functions that is not restricted to be 1-slot type flagManualOnBounce = 32768, // for onInternalMessage, don't insert "if (isBounced) return" }; @@ -212,7 +213,7 @@ struct FunctionData final : Symbol { bool does_mutate_self() const { return (flags & flagAcceptsSelf) && parameters[0].is_mutate_parameter(); } bool is_really_used() const { return flags & flagReallyUsed; } bool is_compile_time_const_val() const { return flags & flagCompileTimeVal; } - bool is_compile_time_special_gen() const { return flags & flagCompileTimeGen; } + bool is_compile_time_special_gen() const { return std::holds_alternative(body); } bool is_variadic_width_T_allowed() const { return flags & flagAllowAnyWidthT; } bool is_manual_on_bounce() const { return flags & flagManualOnBounce; } @@ -299,6 +300,8 @@ struct AliasDefData final : Symbol { struct StructFieldData final : Symbol { int field_idx; + bool is_private; + bool is_readonly; AnyTypeV type_node; TypePtr declared_type = nullptr; // = resolved type_node AnyExprV default_value; // nullptr if no default @@ -309,9 +312,11 @@ struct StructFieldData final : Symbol { void assign_resolved_type(TypePtr declared_type); void assign_default_value(AnyExprV default_value); - StructFieldData(std::string name, SrcLocation loc, int field_idx, AnyTypeV type_node, AnyExprV default_value) + StructFieldData(std::string name, SrcLocation loc, int field_idx, bool is_private, bool is_readonly, AnyTypeV type_node, AnyExprV default_value) : Symbol(std::move(name), loc) , field_idx(field_idx) + , is_private(is_private) + , is_readonly(is_readonly) , type_node(type_node) , default_value(default_value) { } @@ -367,6 +372,41 @@ struct StructData final : Symbol { std::string as_human_readable() const; }; +struct EnumMemberData final : Symbol { + AnyExprV init_value; // nullptr if no init (`Red`, not `Red = 1`) + td::RefInt256 computed_value; // auto-calculated or assigned from init if integer + + bool has_init_value() const { return init_value != nullptr; } + + EnumMemberData(std::string name, SrcLocation loc, AnyExprV init_value) + : Symbol(std::move(name), loc) + , init_value(init_value) { + } + + EnumMemberData* mutate() const { return const_cast(this); } + void assign_init_value(AnyExprV init_value); + void assign_computed_value(td::RefInt256 computed_value); +}; + +struct EnumDefData final : Symbol { + AnyTypeV colon_type_node; // nullptr if no serialization type after `:` + TypePtr colon_type = nullptr; // = resolved colon_type_node + std::vector members; + + EnumMemberPtr find_member(std::string_view member_name) const; + + EnumDefData(std::string name, SrcLocation loc, AnyTypeV colon_type_node, std::vector&& members) + : Symbol(std::move(name), loc) + , colon_type_node(colon_type_node) + , members(std::move(members)) { + } + + std::string as_human_readable() const; + + EnumDefData* mutate() const { return const_cast(this); } + void assign_resolved_colon_type(TypePtr colon_type); +}; + struct TypeReferenceUsedAsSymbol final : Symbol { TypePtr resolved_type; @@ -384,13 +424,8 @@ class GlobalSymbolTable { } public: - void add_function(FunctionPtr f_sym); - void add_global_var(GlobalVarPtr g_sym); - void add_global_const(GlobalConstPtr c_sym); - void add_type_alias(AliasDefPtr a_sym); - void add_struct(StructPtr s_sym); - - void replace_function(FunctionPtr f_sym); + void add_global_symbol(const Symbol* sym); + void add_function(FunctionPtr f_sym) { add_global_symbol(f_sym); } const Symbol* lookup(std::string_view name) const { const auto it = entries.find(key_hash(name)); diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index 77d745eac..1e5f3ba54 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "1.0.0"; +constexpr const char* TOLK_VERSION = "1.1.0"; } // namespace tolk diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 69b03f20a..cef2365ec 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -63,6 +63,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_check_inferred_types(); pipeline_refine_lvalue_for_mutate_arguments(); pipeline_check_rvalue_lvalue(); + pipeline_check_private_fields_usage(); pipeline_check_pure_impure_operations(); pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); diff --git a/tolk/tolk.h b/tolk/tolk.h index 9bef3c4df..13a9acefe 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -907,6 +907,7 @@ struct Optimizer { bool detect_rewrite_SWAP_PUSH_STUR(); bool detect_rewrite_SWAP_STxxxR(); bool detect_rewrite_NOT_THROWIF(); + bool detect_rewrite_DICTSETB_DICTSET(); AsmOpConsList extract_code(); }; @@ -1028,17 +1029,26 @@ struct Stack { * */ -typedef std::function&, std::vector&, SrcLocation)> simple_compile_func_t; +struct FunctionBodyBuiltinAsmOp { + using CompileToAsmOpImpl = AsmOp(std::vector&, std::vector&, SrcLocation); + + std::function simple_compile; -struct FunctionBodyBuiltin { - simple_compile_func_t simple_compile; - - explicit FunctionBodyBuiltin(simple_compile_func_t compile) + explicit FunctionBodyBuiltinAsmOp(std::function compile) : simple_compile(std::move(compile)) {} void compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation loc) const; }; +struct FunctionBodyBuiltinGenerateOps { + using GenerateOpsImpl = std::vector(FunctionPtr, CodeBlob&, SrcLocation, const std::vector>&); + + std::function generate_ops; + + explicit FunctionBodyBuiltinGenerateOps(std::function generate_ops) + : generate_ops(std::move(generate_ops)) {} +}; + struct FunctionBodyAsm { std::vector ops; @@ -1059,6 +1069,12 @@ struct LazyVarRefAtCodegen { : var_ref(var_ref), var_state(var_state) {} }; +// CachedConstValueAtCodegen is used for a map [some_const => '5] +struct CachedConstValueAtCodegen { + GlobalConstPtr const_ref; + std::vector ir_idx; +}; + struct CodeBlob { int var_cnt, in_var_cnt; FunctionPtr fun_ref; @@ -1066,7 +1082,9 @@ struct CodeBlob { SrcLocation forced_loc; std::vector vars; std::vector lazy_variables; + std::vector cached_consts; std::vector* inline_rvect_out = nullptr; + bool inside_evaluating_constant = false; bool inlining_before_immediate_return = false; std::unique_ptr ops; std::unique_ptr* cur_ops; @@ -1121,6 +1139,7 @@ struct CodeBlob { } const LazyVariableLoadedState* get_lazy_variable(LocalVarPtr var_ref) const; const LazyVariableLoadedState* get_lazy_variable(AnyExprV v) const; + const CachedConstValueAtCodegen* get_cached_const(GlobalConstPtr const_ref) const; void prune_unreachable_code(); void fwd_analyze(); void mark_noreturn(); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index cf89edb0b..ba796defd 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -24,109 +24,58 @@ namespace tolk { /* - * This class stores a big hashtable [hash => TypePtr] - * Every non-trivial TypeData*::create() method at first looks here, and allocates an object only if not found. - * That's why all allocated TypeData objects are unique, and can be compared as pointers. - * But compare pointers carefully due to type aliases. + * Every TypeData has children_flags (just a mask of all children), + * here we have an utility to calculate them at creation. */ -class TypeDataHasherForUnique { - uint64_t cur_hash; +class CalcChildrenFlags { int children_flags_mask = 0; - static std::unordered_map all_unique_occurred_types; - public: - explicit TypeDataHasherForUnique(uint64_t initial_arbitrary_unique_number) - : cur_hash(initial_arbitrary_unique_number) {} - - void feed_hash(uint64_t val) { - cur_hash = cur_hash * 56235515617499ULL + val; - } - - void feed_string(const std::string& s) { - feed_hash(std::hash{}(s)); - } - void feed_child(TypePtr inner) { - feed_hash(reinterpret_cast(inner)); children_flags_mask |= inner->flags; } - int children_flags() const { - return children_flags_mask; - } - - GNU_ATTRIBUTE_FLATTEN - TypePtr get_existing() const { - auto it = all_unique_occurred_types.find(cur_hash); - return it != all_unique_occurred_types.end() ? it->second : nullptr; + void feed_child(const std::vector& children) { + for (TypePtr inner : children) { + children_flags_mask |= inner->flags; + } } - GNU_ATTRIBUTE_NOINLINE - TypePtr register_unique(TypePtr newly_created) const { -#ifdef TOLK_DEBUG - assert(all_unique_occurred_types.find(cur_hash) == all_unique_occurred_types.end()); -#endif - all_unique_occurred_types[cur_hash] = newly_created; - return newly_created; + int children_flags() const { + return children_flags_mask; } }; /* * This class stores a hashtable [TypePtr => type_id] * We need type_id to support union types, that are stored as tagged unions on a stack. - * Every type that can be contained inside a union, has type_id. + * Every type actually contained inside a union, has type_id. * Some type_id are predefined (1 = int, etc.), but all user-defined types are assigned type_id. */ class TypeIdCalculation { static int last_type_id; static std::unordered_map map_ptr_to_type_id; - static std::vector instantiated_structs; public: static int assign_type_id(TypePtr self) { - if (self->has_type_alias_inside()) { // type_id is calculated without aliases - self = unwrap_type_alias_deeply(self); // `(int,int)` equals `(IntAlias,IntAlias)`. - } - if (auto it = map_ptr_to_type_id.find(self); it != map_ptr_to_type_id.end()) { + // type_id is calculated without aliases, based on "equal to"; + // for instance, `UserId` / `OwnerId` / `int` will have the same type_id without any runtime conversion + auto it = std::find_if(map_ptr_to_type_id.begin(), map_ptr_to_type_id.end(), [self](std::pair existing) { + return existing.first->equal_to(self); + }); + if (it != map_ptr_to_type_id.end()) { return it->second; } - // make `Wrapper>` and `Wrapper>` and `Wrapper` have equal type_id - if (const TypeDataStruct* t_struct = self->try_as(); t_struct && t_struct->struct_ref->is_instantiation_of_generic_struct()) { - StructPtr struct_ref = t_struct->struct_ref; - instantiated_structs.push_back(struct_ref); - for (StructPtr another_ref : instantiated_structs) { - if (struct_ref != another_ref && self->equal_to(TypeDataStruct::create(another_ref))) { - auto it = map_ptr_to_type_id.find(TypeDataStruct::create(another_ref)); - assert(it != map_ptr_to_type_id.end()); - int type_id = it->second; - map_ptr_to_type_id[self] = type_id; - return type_id; - } - } - } - int type_id = ++last_type_id; map_ptr_to_type_id[self] = type_id; return type_id; } - - static TypePtr unwrap_type_alias_deeply(TypePtr type) { - return type->replace_children_custom([](TypePtr child) { - if (const TypeDataAlias* as_alias = child->try_as()) { - return as_alias->underlying_type->unwrap_alias(); - } - return child; - }); - } }; int TypeIdCalculation::last_type_id = 128; // below 128 reserved for built-in types -std::unordered_map TypeDataHasherForUnique::all_unique_occurred_types; std::unordered_map TypeIdCalculation::map_ptr_to_type_id; -std::vector TypeIdCalculation::instantiated_structs; TypePtr TypeDataInt::singleton; TypePtr TypeDataBool::singleton; @@ -159,31 +108,6 @@ void type_system_init() { } -bool TypeData::equal_to_slow_path(TypePtr lhs, TypePtr rhs) { - if (lhs->has_type_alias_inside()) { - lhs = TypeIdCalculation::unwrap_type_alias_deeply(lhs); - } - if (rhs->has_type_alias_inside()) { - rhs = TypeIdCalculation::unwrap_type_alias_deeply(rhs); - } - if (lhs == rhs) { - return true; - } - - if (const TypeDataUnion* lhs_union = lhs->try_as()) { - if (const TypeDataUnion* rhs_union = rhs->try_as()) { - return lhs_union->variants.size() == rhs_union->variants.size() && lhs_union->has_all_variants_of(rhs_union); - } - } - if (const TypeDataStruct* lhs_struct = lhs->try_as(); lhs_struct && lhs_struct->struct_ref->is_instantiation_of_generic_struct()) { - if (const TypeDataStruct* rhs_struct = rhs->try_as(); rhs_struct && rhs_struct->struct_ref->is_instantiation_of_generic_struct()) { - return lhs_struct->struct_ref->base_struct_ref == rhs_struct->struct_ref->base_struct_ref - && lhs_struct->struct_ref->substitutedTs->equal_to(rhs_struct->struct_ref->substitutedTs); - } - } - return false; -} - TypePtr TypeData::unwrap_alias_slow_path(TypePtr lhs) { TypePtr unwrapped = lhs; while (const TypeDataAlias* as_alias = unwrapped->try_as()) { @@ -192,6 +116,27 @@ TypePtr TypeData::unwrap_alias_slow_path(TypePtr lhs) { return unwrapped; } +// having `type UserId = int` and `type OwnerId = int` (when their underlying types are equal), +// make `UserId` and `OwnerId` NOT equal and NOT assignable (although they'll have the same type_id); +// it allows overloading methods for these types independently, e.g. +// > type BalanceList = dict +// > type AssetList = dict +// > fun BalanceList.validate(self) +// > fun AssetList.validate(self) +static bool are_two_equal_type_aliases_different(const TypeDataAlias* t1, const TypeDataAlias* t2) { + if (t1->alias_ref == t2->alias_ref) { + return false; + } + if (t1->alias_ref->is_instantiation_of_generic_alias() && t2->alias_ref->is_instantiation_of_generic_alias()) { + return !t1->alias_ref->substitutedTs->equal_to(t2->alias_ref->substitutedTs); + } + // handle `type MInt2 = MInt1`, as well as `type BalanceList = dict`, then they are equal + const TypeDataAlias* t_und1 = t1->underlying_type->try_as(); + const TypeDataAlias* t_und2 = t2->underlying_type->try_as(); + bool one_aliases_another = (t_und1 && t_und1->alias_ref == t2->alias_ref) + || (t_und2 && t1->alias_ref == t_und2->alias_ref); + return !one_aliases_another; +} // -------------------------------------------- // create() @@ -202,152 +147,71 @@ TypePtr TypeData::unwrap_alias_slow_path(TypePtr lhs) { // TypePtr TypeDataAlias::create(AliasDefPtr alias_ref) { - TypeDataHasherForUnique hash(5694590762732189561ULL); - hash.feed_string(alias_ref->name); - hash.feed_child(alias_ref->underlying_type); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - TypePtr underlying_type = alias_ref->underlying_type; if (underlying_type == TypeDataNullLiteral::create() || underlying_type == TypeDataNever::create() || underlying_type == TypeDataVoid::create()) { return underlying_type; // aliasing these types is strange, don't store an alias } - return hash.register_unique(new TypeDataAlias(hash.children_flags(), alias_ref, underlying_type)); + CalcChildrenFlags reg; + reg.feed_child(alias_ref->underlying_type); + return new TypeDataAlias(reg.children_flags(), alias_ref, underlying_type); } TypePtr TypeDataFunCallable::create(std::vector&& params_types, TypePtr return_type) { - TypeDataHasherForUnique hash(3184039965511020991ULL); - for (TypePtr param : params_types) { - hash.feed_child(param); - hash.feed_hash(767721); - } - hash.feed_child(return_type); - hash.feed_hash(767722); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataFunCallable(hash.children_flags(), std::move(params_types), return_type)); + CalcChildrenFlags reg; + reg.feed_child(params_types); + reg.feed_child(return_type); + return new TypeDataFunCallable(reg.children_flags(), std::move(params_types), return_type); } TypePtr TypeDataGenericT::create(std::string&& nameT) { - TypeDataHasherForUnique hash(9145033724911680012ULL); - hash.feed_string(nameT); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataGenericT(std::move(nameT))); + return new TypeDataGenericT(std::move(nameT)); } TypePtr TypeDataGenericTypeWithTs::create(StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments) { - TypeDataHasherForUnique hash(7451094818554079348ULL); // use it only to calculate children_flags if (struct_ref) { assert(alias_ref == nullptr && struct_ref->is_generic_struct()); - hash.feed_string(struct_ref->name); } else { assert(struct_ref == nullptr && alias_ref->is_generic_alias()); - hash.feed_string(alias_ref->name); - } - for (TypePtr argT : type_arguments) { - hash.feed_child(argT); } - return new TypeDataGenericTypeWithTs(hash.children_flags(), struct_ref, alias_ref, std::move(type_arguments)); + CalcChildrenFlags reg; + reg.feed_child(type_arguments); + return new TypeDataGenericTypeWithTs(reg.children_flags(), struct_ref, alias_ref, std::move(type_arguments)); } TypePtr TypeDataStruct::create(StructPtr struct_ref) { - TypeDataHasherForUnique hash(8315986401043319583ULL); - hash.feed_string(struct_ref->name); + return new TypeDataStruct(struct_ref); +} - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataStruct(struct_ref)); +TypePtr TypeDataEnum::create(EnumDefPtr enum_ref) { + return new TypeDataEnum(enum_ref); } TypePtr TypeDataTensor::create(std::vector&& items) { - TypeDataHasherForUnique hash(3159238551239480381ULL); - for (TypePtr item : items) { - hash.feed_child(item); - hash.feed_hash(819613); - } - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataTensor(hash.children_flags(), std::move(items))); + CalcChildrenFlags reg; + reg.feed_child(items); + return new TypeDataTensor(reg.children_flags(), std::move(items)); } TypePtr TypeDataBrackets::create(std::vector&& items) { - TypeDataHasherForUnique hash(9189266157349499320ULL); - for (TypePtr item : items) { - hash.feed_child(item); - hash.feed_hash(735911); - } - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataBrackets(hash.children_flags(), std::move(items))); + CalcChildrenFlags reg; + reg.feed_child(items); + return new TypeDataBrackets(reg.children_flags(), std::move(items)); } TypePtr TypeDataIntN::create(int n_bits, bool is_unsigned, bool is_variadic) { - TypeDataHasherForUnique hash(1678330938771108027ULL); - hash.feed_hash(n_bits); - hash.feed_hash(is_unsigned); - hash.feed_hash(is_variadic); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataIntN(n_bits, is_unsigned, is_variadic)); + return new TypeDataIntN(n_bits, is_unsigned, is_variadic); } TypePtr TypeDataBitsN::create(int n_width, bool is_bits) { - TypeDataHasherForUnique hash(7810988137199333041ULL); - hash.feed_hash(n_width); - hash.feed_hash(is_bits); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataBitsN(n_width, is_bits)); + return new TypeDataBitsN(n_width, is_bits); } TypePtr TypeDataUnion::create(std::vector&& variants) { - TypeDataHasherForUnique hash(8719233194368471403ULL); - for (TypePtr variant : variants) { - hash.feed_child(variant); - hash.feed_hash(817663); - } - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - - // for generics, like `f(v: T?)`, no type_id exists - // in this case, don't try to flatten: we have no info - // after instantiation, a new union type (with resolved variants) will be created - bool not_ready_yet = false; - for (TypePtr variant : variants) { - not_ready_yet |= variant->has_genericT_inside(); - } - if (not_ready_yet) { - TypePtr or_null = nullptr; - if (variants.size() == 2) { - if (variants[0] == TypeDataNullLiteral::create() || variants[1] == TypeDataNullLiteral::create()) { - or_null = variants[variants[0] == TypeDataNullLiteral::create()]; - } - } - return hash.register_unique(new TypeDataUnion(hash.children_flags(), or_null, std::move(variants))); - } - // flatten variants and remove duplicates - // note, that `int | slice` and `int | int | slice` are different TypePtr, but actually the same variants + // note, that `int | slice` and `int | int | slice` are different TypePtr, but actually the same variants; + // note, that `UserId | OwnerId` (both are aliases to `int`) will emit `UserId` (OwnerId is a duplicate) std::vector flat_variants; flat_variants.reserve(variants.size()); for (TypePtr variant : variants) { @@ -370,21 +234,17 @@ TypePtr TypeDataUnion::create(std::vector&& variants) { if (flat_variants.size() == 1) { // `int | int` return flat_variants[0]; } - return hash.register_unique(new TypeDataUnion(hash.children_flags(), or_null, std::move(flat_variants))); -} -TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { - // calculate exactly the same hash as for `T | null` to create std::vector only if type seen the first time - TypeDataHasherForUnique hash(8719233194368471403ULL); - hash.feed_child(nullable); - hash.feed_hash(817663); - hash.feed_child(TypeDataNullLiteral::create()); - hash.feed_hash(817663); + CalcChildrenFlags reg; + reg.feed_child(flat_variants); + return new TypeDataUnion(reg.children_flags(), or_null, std::move(flat_variants)); +} - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return create({nullable, TypeDataNullLiteral::create()}); +TypePtr TypeDataMapKV::create(TypePtr TKey, TypePtr TValue) { + CalcChildrenFlags reg; + reg.feed_child(TKey); + reg.feed_child(TValue); + return new TypeDataMapKV(reg.children_flags(), TKey, TValue); } @@ -482,6 +342,10 @@ int TypeDataStruct::get_type_id() const { return TypeIdCalculation::assign_type_id(this); } +int TypeDataEnum::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + int TypeDataTensor::get_type_id() const { assert(!has_genericT_inside()); return TypeIdCalculation::assign_type_id(this); @@ -513,6 +377,11 @@ int TypeDataUnion::get_type_id() const { throw Fatal("unexpected get_type_id() call"); } +int TypeDataMapKV::get_type_id() const { + assert(!has_genericT_inside()); + return TypeIdCalculation::assign_type_id(this); +} + int TypeDataUnknown::get_type_id() const { assert(false); // unknown can not be inside a union throw Fatal("unexpected get_type_id() call"); @@ -560,6 +429,10 @@ std::string TypeDataStruct::as_human_readable() const { return struct_ref->name; } +std::string TypeDataEnum::as_human_readable() const { + return enum_ref->name; +} + std::string TypeDataTensor::as_human_readable() const { std::string result = "("; for (TypePtr item : items) { @@ -620,6 +493,10 @@ std::string TypeDataUnion::as_human_readable() const { return result; } +std::string TypeDataMapKV::as_human_readable() const { + return "map<" + TKey->as_human_readable() + ", " + TValue->as_human_readable() + ">"; +} + // -------------------------------------------- // replace_children_custom() @@ -674,6 +551,10 @@ TypePtr TypeDataUnion::replace_children_custom(const ReplacerCallbackT& callback return callback(create(std::move(mapped))); } +TypePtr TypeDataMapKV::replace_children_custom(const ReplacerCallbackT& callback) const { + return callback(create(TKey->replace_children_custom(callback), TValue->replace_children_custom(callback))); +} + // -------------------------------------------- // can_rhs_be_assigned() @@ -685,14 +566,18 @@ TypePtr TypeDataUnion::replace_children_custom(const ReplacerCallbackT& callback // bool TypeDataAlias::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + // having `type UserId = int` and `type OwnerId = int`, make them NOT assignable without `as` + // (although they both have the same type_id) + if (underlying_type->equal_to(rhs_alias->underlying_type)) { + return !are_two_equal_type_aliases_different(this, rhs_alias); + } } return underlying_type->can_rhs_be_assigned(rhs); } bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (rhs->try_as()) { @@ -708,7 +593,7 @@ bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataBool::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -718,7 +603,7 @@ bool TypeDataBool::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataCell::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataStruct* rhs_struct = rhs->try_as()) { @@ -733,7 +618,7 @@ bool TypeDataCell::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -743,7 +628,7 @@ bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -753,7 +638,7 @@ bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataTuple::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -763,7 +648,7 @@ bool TypeDataTuple::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -773,7 +658,7 @@ bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataAddress::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -783,7 +668,7 @@ bool TypeDataAddress::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { @@ -793,9 +678,6 @@ bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; - } if (const TypeDataFunCallable* rhs_callable = rhs->try_as()) { if (rhs_callable->params_size() != params_size()) { return false; @@ -826,14 +708,21 @@ bool TypeDataGenericTypeWithTs::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataStruct::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; + if (const TypeDataStruct* rhs_struct = rhs->try_as()) { // C> = C + return struct_ref == rhs_struct->struct_ref || equal_to(rhs_struct); } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } - if (const TypeDataStruct* rhs_struct = rhs->try_as()) { // C> = C - return equal_to(rhs_struct); + return rhs == TypeDataNever::create(); +} + +bool TypeDataEnum::can_rhs_be_assigned(TypePtr rhs) const { + if (const TypeDataEnum* rhs_enum = rhs->try_as()) { + return enum_ref == rhs_enum->enum_ref; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); } return rhs == TypeDataNever::create(); } @@ -869,32 +758,33 @@ bool TypeDataBrackets::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; - } if (rhs == TypeDataInt::create()) { return true; } + if (const TypeDataIntN* rhs_intN = rhs->try_as()) { + // `int8` is NOT assignable to `int32` without `as` + return n_bits == rhs_intN->n_bits && is_unsigned == rhs_intN->is_unsigned && is_variadic == rhs_intN->is_variadic; + } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } - return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` + return rhs == TypeDataNever::create(); } bool TypeDataBitsN::can_rhs_be_assigned(TypePtr rhs) const { - // `slice` is NOT assignable to bitsN without `as` - // `bytes32` is NOT assignable to `bytes256` and even to `bits256` without `as` - if (rhs == this) { - return true; + if (const TypeDataBitsN* rhs_bitsN = rhs->try_as()) { + // `slice` is NOT assignable to bitsN without `as` + // `bytes32` is NOT assignable to `bytes256` and even to `bits256` without `as` + return n_width == rhs_bitsN->n_width && is_bits == rhs_bitsN->is_bits; } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } - return false; + return rhs == TypeDataNever::create(); } bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } if (rhs == TypeDataInt::create()) { @@ -907,9 +797,6 @@ bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataUnion::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; - } if (calculate_exact_variant_to_fit_rhs(rhs)) { // `int` to `int | slice`, `int?` to `int8?`, `(int, null)` to `(int, T?) | slice` return true; } @@ -919,7 +806,16 @@ bool TypeDataUnion::can_rhs_be_assigned(TypePtr rhs) const { if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } + return rhs == TypeDataNever::create(); +} +bool TypeDataMapKV::can_rhs_be_assigned(TypePtr rhs) const { + if (const TypeDataMapKV* rhs_map = rhs->try_as()) { + return TKey->equal_to(rhs_map->TKey) && TValue->equal_to(rhs_map->TValue); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -928,11 +824,11 @@ bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataNever::can_rhs_be_assigned(TypePtr rhs) const { - return rhs == TypeDataNever::create(); + return rhs == singleton; } bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { + if (rhs == singleton) { return true; } return rhs == TypeDataNever::create(); @@ -974,10 +870,13 @@ bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to == TypeDataCoins::create()) { // `int` as `coins` return true; } + if (cast_to->try_as()) { // `int` as `Color` (all enums are integer) + return true; + } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -993,7 +892,7 @@ bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1006,7 +905,7 @@ bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1022,7 +921,7 @@ bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1032,7 +931,7 @@ bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1042,7 +941,7 @@ bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1052,11 +951,11 @@ bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataAddress::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (cast_to == TypeDataSlice::create()) { + if (cast_to == TypeDataSlice::create() || cast_to->try_as()) { return true; } if (const TypeDataUnion* to_union = cast_to->try_as()) { @@ -1065,7 +964,7 @@ bool TypeDataAddress::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1075,7 +974,7 @@ bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this; + return cast_to == singleton; } bool TypeDataFunCallable::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1124,7 +1023,23 @@ bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataStruct* to_struct = cast_to->try_as()) { // C> as C return equal_to(to_struct); } - return cast_to == this; + return false; +} + +bool TypeDataEnum::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to == TypeDataInt::create()) { + return true; + } + if (cast_to->try_as()) { + return true; // all enums are integers, they can be `as` cast to each other + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return false; } bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1205,7 +1120,7 @@ bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to == TypeDataInt::create()) { return true; } - return cast_to == this; + return cast_to == singleton; } bool TypeDataUnion::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -1221,6 +1136,19 @@ bool TypeDataUnion::can_be_casted_with_as_operator(TypePtr cast_to) const { return false; } +bool TypeDataMapKV::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const TypeDataMapKV* to_map = cast_to->try_as()) { + return TKey->equal_to(to_map->TKey) && TValue->equal_to(to_map->TValue); + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return false; +} + bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { // 'unknown' can be cast to any TVM value return cast_to->get_width_on_stack() == 1; @@ -1231,7 +1159,7 @@ bool TypeDataNever::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataVoid::can_be_casted_with_as_operator(TypePtr cast_to) const { - return cast_to == this; + return cast_to == singleton; } @@ -1279,6 +1207,10 @@ bool TypeDataUnion::can_hold_tvm_null_instead() const { return or_null && !or_null->can_hold_tvm_null_instead(); } +bool TypeDataMapKV::can_hold_tvm_null_instead() const { + return false; // map is an optional cell, so `map?` requires a nullable presence slot +} + bool TypeDataNever::can_hold_tvm_null_instead() const { return false; } @@ -1288,12 +1220,167 @@ bool TypeDataVoid::can_hold_tvm_null_instead() const { } +// -------------------------------------------- +// equal_to() +// +// comparing types for equality (when implementation differs from a default "compare pointers"); +// two types are EQUAL is a much more strict property than "assignable"; +// a union type can hold only non-equal types; for instance, having `type MyInt = int`, a union `int | MyInt` == `int`; +// searching for a compatible method for a receiver is also based on equal_to() as first priority +// + +bool TypeDataAlias::equal_to(TypePtr rhs) const { + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + // given `type UserId = int` and `type OwnerId = int`, treat them as NOT equal (they are also not assignable); + // (but nevertheless, they will have the same type_id, and `UserId | OwnerId` is not a valid union) + if (underlying_type->equal_to(rhs_alias->underlying_type)) { + return !are_two_equal_type_aliases_different(this, rhs_alias); + } + } + return underlying_type->equal_to(rhs); +} + +bool TypeDataFunCallable::equal_to(TypePtr rhs) const { + if (const TypeDataFunCallable* rhs_callable = rhs->try_as(); rhs_callable && rhs_callable->params_size() == params_size()) { + for (int i = 0; i < params_size(); ++i) { + if (!params_types[i]->equal_to(rhs_callable->params_types[i])) { + return false; + } + } + return return_type->equal_to(rhs_callable->return_type); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataGenericT::equal_to(TypePtr rhs) const { + if (const TypeDataGenericT* rhs_T = rhs->try_as()) { + return nameT == rhs_T->nameT; + } + return false; +} + +bool TypeDataGenericTypeWithTs::equal_to(TypePtr rhs) const { + if (const TypeDataGenericTypeWithTs* rhs_Ts = rhs->try_as(); rhs_Ts && size() == rhs_Ts->size()) { + for (int i = 0; i < size(); ++i) { + if (!type_arguments[i]->equal_to(rhs_Ts->type_arguments[i])) { + return false; + } + } + return alias_ref == rhs_Ts->alias_ref && struct_ref == rhs_Ts->struct_ref; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataStruct::equal_to(TypePtr rhs) const { + if (const TypeDataStruct* rhs_struct = rhs->try_as()) { + if (struct_ref == rhs_struct->struct_ref) { + return true; + } + if (struct_ref->is_instantiation_of_generic_struct() && rhs_struct->struct_ref->is_instantiation_of_generic_struct()) { + return struct_ref->base_struct_ref == rhs_struct->struct_ref->base_struct_ref + && struct_ref->substitutedTs->equal_to(rhs_struct->struct_ref->substitutedTs); + } + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataEnum::equal_to(TypePtr rhs) const { + if (const TypeDataEnum* rhs_enum = rhs->try_as()) { + return enum_ref == rhs_enum->enum_ref; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataTensor::equal_to(TypePtr rhs) const { + if (const TypeDataTensor* rhs_tensor = rhs->try_as(); rhs_tensor && size() == rhs_tensor->size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->equal_to(rhs_tensor->items[i])) { + return false; + } + } + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataBrackets::equal_to(TypePtr rhs) const { + if (const TypeDataBrackets* rhs_brackets = rhs->try_as(); rhs_brackets && size() == rhs_brackets->size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->equal_to(rhs_brackets->items[i])) { + return false; + } + } + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataIntN::equal_to(TypePtr rhs) const { + if (const TypeDataIntN* rhs_intN = rhs->try_as()) { + return n_bits == rhs_intN->n_bits && is_unsigned == rhs_intN->is_unsigned && is_variadic == rhs_intN->is_variadic; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataBitsN::equal_to(TypePtr rhs) const { + if (const TypeDataBitsN* rhs_bitsN = rhs->try_as()) { + return n_width == rhs_bitsN->n_width && is_bits == rhs_bitsN->is_bits; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataUnion::equal_to(TypePtr rhs) const { + if (const TypeDataUnion* rhs_union = rhs->try_as()) { + return variants.size() == rhs_union->variants.size() && has_all_variants_of(rhs_union); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + +bool TypeDataMapKV::equal_to(TypePtr rhs) const { + if (const TypeDataMapKV* rhs_map = rhs->try_as()) { + return TKey->equal_to(rhs_map->TKey) && TValue->equal_to(rhs_map->TValue); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return equal_to(rhs_alias->underlying_type); + } + return false; +} + + // union types creation is a bit tricky: nested unions are flattened, duplicates are removed -// so, a resolved union type has variants, each with unique type_id -// (type_id is calculated with aliases erasure) +// so, a resolved union type has variants, each will be assigned a unique type_id (tagged unions) void TypeDataUnion::append_union_type_variant(TypePtr variant, std::vector& out_unique_variants) { + // having `UserId | OwnerId` (both are aliases to `int`) merge them into just `UserId`, because underlying are equal + TypePtr underlying_variant = variant->unwrap_alias(); for (TypePtr existing : out_unique_variants) { - if (existing->get_type_id() == variant->get_type_id()) { + if (existing->equal_to(underlying_variant)) { return; } } @@ -1301,9 +1388,9 @@ void TypeDataUnion::append_union_type_variant(TypePtr variant, std::vectorget_type_id() == type_id) { + if (self_variant->equal_to(rhs_type)) { return true; } } @@ -1312,7 +1399,7 @@ bool TypeDataUnion::has_variant_with_type_id(int type_id) const { bool TypeDataUnion::has_all_variants_of(const TypeDataUnion* rhs_type) const { for (TypePtr rhs_variant : rhs_type->variants) { - if (!has_variant_with_type_id(rhs_variant->get_type_id())) { + if (!has_variant_equal_to(rhs_variant)) { return false; } } @@ -1339,7 +1426,7 @@ TypePtr TypeDataUnion::calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) cons } // `int` to `int | int8` is okay: exact type matching for (TypePtr variant : variants) { - if (variant->get_type_id() == rhs_type->get_type_id()) { + if (variant->equal_to(rhs_type)) { return variant; } } diff --git a/tolk/type-system.h b/tolk/type-system.h index e9a63c213..be043a643 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -42,20 +42,20 @@ class TypeData { // bits of flag_mask, to store often-used properties and return them without tree traversing const int flags; - friend class TypeDataHasherForUnique; + friend class CalcChildrenFlags; protected: enum flag_mask { flag_contains_unknown_inside = 1 << 1, flag_contains_genericT_inside = 1 << 2, flag_contains_type_alias_inside = 1 << 3, + flag_contains_mapKV_inside = 1 << 4, }; explicit TypeData(int flags_with_children) : flags(flags_with_children) { } - static bool equal_to_slow_path(TypePtr lhs, TypePtr rhs); static TypePtr unwrap_alias_slow_path(TypePtr lhs); public: @@ -71,9 +71,6 @@ class TypeData { return 1; // most types occupy 1 stack slot (int, cell, slice, etc.) } - bool equal_to(TypePtr rhs) const { - return this == rhs || equal_to_slow_path(this, rhs); - } TypePtr unwrap_alias() const { return has_type_alias_inside() ? unwrap_alias_slow_path(this) : this; } @@ -81,6 +78,7 @@ class TypeData { bool has_unknown_inside() const { return flags & flag_contains_unknown_inside; } bool has_genericT_inside() const { return flags & flag_contains_genericT_inside; } bool has_type_alias_inside() const { return flags & flag_contains_type_alias_inside; } + bool has_mapKV_inside() const { return flags & flag_contains_mapKV_inside; } using ReplacerCallbackT = std::function; @@ -93,6 +91,10 @@ class TypeData { return true; } + virtual bool equal_to(TypePtr rhs) const { + return this == rhs->unwrap_alias(); + } + virtual TypePtr replace_children_custom(const ReplacerCallbackT& callback) const { return callback(this); } @@ -124,6 +126,7 @@ class TypeDataAlias final : public TypeData { bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; bool can_hold_tvm_null_instead() const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -320,6 +323,7 @@ class TypeDataFunCallable final : public TypeData { bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -343,6 +347,7 @@ class TypeDataGenericT final : public TypeData { std::string as_human_readable() const override { return nameT; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -373,6 +378,7 @@ class TypeDataGenericTypeWithTs final : public TypeData { bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -398,6 +404,28 @@ class TypeDataStruct final : public TypeData { bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; bool can_hold_tvm_null_instead() const override; + bool equal_to(TypePtr rhs) const override; +}; + +/* + * `Color.Red`, `BounceMode.NoBounce` is TypeDataEnum. At TVM level, it's `int`. + * Its value is either assigned like `Red = 1` or auto-calculated. + */ +class TypeDataEnum final : public TypeData { + explicit TypeDataEnum(EnumDefPtr enum_ref) + : TypeData(0) + , enum_ref(enum_ref) {} + +public: + EnumDefPtr enum_ref; + + static TypePtr create(EnumDefPtr enum_ref); + + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -425,6 +453,7 @@ class TypeDataTensor final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -449,6 +478,7 @@ class TypeDataBrackets final : public TypeData { bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -475,6 +505,7 @@ class TypeDataIntN final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -518,6 +549,7 @@ class TypeDataBitsN final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool equal_to(TypePtr rhs) const override; }; /* @@ -528,6 +560,8 @@ class TypeDataBitsN final : public TypeData { * - `T | null`, if T is 1 slot (like `int | null`), then it's still 1 slot * - `T | null`, if T is N slots (like `(int, int)?`), it's stored as N+1 slots (the last for type_id if T or 0 if null) * - `T1 | T2 | ...` is a tagged union: occupy max(T_i)+1 slots (1 for type_id) + * When a union is created, variants are flattened, duplicates are removed: `int | UserId | IntOrSlice` = `int | slice`, + * duplicates are detected based on `equal_to()`, and a union can be tested on having `has_variant_equal_to()`. */ class TypeDataUnion final : public TypeData { TypeDataUnion(int children_flags, TypePtr or_null, std::vector&& variants) @@ -535,7 +569,6 @@ class TypeDataUnion final : public TypeData { , or_null(or_null) , variants(std::move(variants)) {} - bool has_variant_with_type_id(int type_id) const; static void append_union_type_variant(TypePtr variant, std::vector& out_unique_variants); public: @@ -543,7 +576,7 @@ class TypeDataUnion final : public TypeData { const std::vector variants; // T_i, flattened, no duplicates; may include aliases, but not other unions static TypePtr create(std::vector&& variants); - static TypePtr create_nullable(TypePtr nullable); + static TypePtr create_nullable(TypePtr nullable) { return create({nullable, TypeDataNullLiteral::create()}); } int size() const { return static_cast(variants.size()); } @@ -551,23 +584,14 @@ class TypeDataUnion final : public TypeData { // true : `int?`, `slice?`, `StructWith1IntField?` // false: `(int, int)?`, `ComplexStruct?`, `()?` bool is_primitive_nullable() const { - return get_width_on_stack() == 1 && or_null != nullptr && or_null->get_width_on_stack() == 1; + return !has_genericT_inside() && get_width_on_stack() == 1 && or_null != nullptr && or_null->get_width_on_stack() == 1; } bool has_null() const { - if (or_null) { - return true; - } - return has_variant_with_type_id(0); - } - bool has_variant_with_type_id(TypePtr rhs_type) const { - int type_id = rhs_type->get_type_id(); - if (or_null) { - return type_id == 0 || type_id == or_null->get_type_id(); - } - return has_variant_with_type_id(type_id); + return or_null != nullptr || has_variant_equal_to(TypeDataNullLiteral::create()); } TypePtr calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const; + bool has_variant_equal_to(TypePtr rhs_type) const; bool has_all_variants_of(const TypeDataUnion* rhs_type) const; int get_variant_idx(TypePtr lookup_variant) const; @@ -578,6 +602,34 @@ class TypeDataUnion final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; + bool equal_to(TypePtr rhs) const override; +}; + +/* + * `map` is a built-in type, a high-level wrapper over TVM dictionaries. + * Internally, a map is just a nullable cell: null represents an empty dict, otherwise it points to a root cell. + * The compiler checks that key-value types are correct, generates optimal TVM instructions, + * auto-serialization of V and non-primitive K to slices, etc. Deeply integrated with stdlib. + */ +class TypeDataMapKV final : public TypeData { + TypeDataMapKV(int children_flags, TypePtr TKey, TypePtr TValue) + : TypeData(children_flags | flag_contains_mapKV_inside) + , TKey(TKey) + , TValue(TValue) {} + +public: + TypePtr TKey; + TypePtr TValue; + + static TypePtr create(TypePtr TKey, TypePtr TValue); + + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool can_hold_tvm_null_instead() const override; + bool equal_to(TypePtr rhs) const override; }; /* diff --git a/ton/ton-types.h b/ton/ton-types.h index 67f4ea203..87947d027 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -411,6 +411,8 @@ struct Ed25519_PrivateKey { struct Ed25519_PublicKey { Bits256 _pubkey; + Ed25519_PublicKey() : _pubkey(td::Bits256::zero()) { + } explicit Ed25519_PublicKey(const Bits256& x) : _pubkey(x) { } explicit Ed25519_PublicKey(const td::ConstBitPtr x) : _pubkey(x) { diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index e972d84e0..45d6c9ebb 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -93,8 +93,9 @@ td::Status LastConfig::process_config_proof(ton::ton_api::object_ptrstate_proof_.as_slice(), raw_config->config_proof_.as_slice())); - TRY_RESULT(config, block::ConfigInfo::extract_config( - std::move(state), block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needCapabilities)); + TRY_RESULT(config, + block::ConfigInfo::extract_config( + std::move(state), blkid, block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needCapabilities)); for (auto i : params_) { VLOG(last_config) << "ConfigParam(" << i << ") = "; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 8a2b47da4..d28cb805d 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -2175,7 +2175,7 @@ class RunEmulator : public TonlibQueryActor { } try { - auto r_config = block::ConfigInfo::extract_config(mc_state_root_, 0b11'11111111); + auto r_config = block::ConfigInfo::extract_config(mc_state_root_, block_id_.mc, 0b11'11111111); if (r_config.is_error()) { check(r_config.move_as_error()); return; @@ -3232,15 +3232,12 @@ struct ToRawTransactions { TRY_RESULT(src, to_std_address(msg_info.src)); TRY_RESULT(dest, to_std_address(msg_info.dest)); TRY_RESULT(fwd_fee, to_balance(msg_info.fwd_fee)); - TRY_RESULT(ihr_fee, to_balance(msg_info.ihr_fee)); auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( - msg_hash, - tonlib_api::make_object(src), - tonlib_api::make_object(std::move(dest)), balance, - std::move(extra_currencies), fwd_fee, ihr_fee, created_lt, std::move(body_hash), - get_data(src)); + msg_hash, tonlib_api::make_object(src), + tonlib_api::make_object(std::move(dest)), balance, std::move(extra_currencies), + fwd_fee, /* ihr_fee = */ 0, created_lt, std::move(body_hash), get_data(src)); } case block::gen::CommonMsgInfo::ext_in_msg_info: { block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; diff --git a/tvm-python/CMakeLists.txt b/tvm-python/CMakeLists.txt index 36fe5e9e0..40bc148e5 100644 --- a/tvm-python/CMakeLists.txt +++ b/tvm-python/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) set(OPENSSL_USE_STATIC_LIBS TRUE) if (NOT Python_FOUND) diff --git a/tvm-python/PyStack.h b/tvm-python/PyStack.h index 5a9459895..70f3ed423 100644 --- a/tvm-python/PyStack.h +++ b/tvm-python/PyStack.h @@ -92,6 +92,10 @@ class PyStackEntry { std::vector as_tuple() { auto x = entry.as_tuple(); + if (x.is_null()) { + throw std::invalid_argument("Stack is not correct type"); + } + std::vector tmp; for (const auto& e : *x) { tmp.push_back(PyStackEntry(e)); @@ -102,11 +106,19 @@ class PyStackEntry { PyContinuation as_cont() { auto x = entry.as_cont(); + if (x.is_null()) { + throw std::invalid_argument("Stack is not correct type"); + } + return PyContinuation(x); } PyCellSlice as_cell_slice() { auto x = entry.as_slice(); + if (x.is_null()) { + throw std::invalid_argument("Stack is not correct type"); + } + vm::CellBuilder cb; cb.append_cellslice(x); td::Ref cell = cb.finalize(x->is_special()); @@ -116,6 +128,10 @@ class PyStackEntry { PyCellBuilder as_cell_builder() { auto x = entry.as_builder(); + if (x.is_null()) { + throw std::invalid_argument("Stack is not correct type"); + } + bool special; auto cs = vm::load_cell_slice_special(x->finalize_copy(), special); return PyCellBuilder(cs); @@ -123,11 +139,15 @@ class PyStackEntry { std::string as_int() { auto x = entry.as_int(); + if (x.is_null()) { + throw std::invalid_argument("Stack is not correct type"); + } + return x->to_dec_string(); } std::string as_string() { - return entry.as_string(); + return std::move(entry.as_string()); } PyCell serialize(int mode = 0) { diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp index a9baa7595..e89cb1bbf 100644 --- a/utils/proxy-liteserver.cpp +++ b/utils/proxy-liteserver.cpp @@ -184,7 +184,7 @@ class ProxyLiteserver : public td::actor::Actor { for (size_t i = 0; i < servers_.size(); ++i) { Server& server = servers_[i]; - server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.hostname, std::make_unique(actor_id(this), i)); server.alive = false; } @@ -197,7 +197,7 @@ class ProxyLiteserver : public td::actor::Actor { } server.alive = ready; LOG(WARNING) << (ready ? "Connected to" : "Disconnected from") << " server #" << idx << " (" - << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + << server.config.hostname << ")"; } void create_ext_server() { @@ -295,8 +295,7 @@ class ProxyLiteserver : public td::actor::Actor { Server& server = servers_[server_idx]; LOG(INFO) << "Sending query " << query_info.to_str() << (wait_mc_seqno_obj ? PSTRING() << " (wait seqno " << wait_mc_seqno_obj->seqno_ << ")" : "") - << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.addr.get_ip_str() - << ":" << server.config.addr.get_port() << ")"; + << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.hostname << ")"; BlockSeqno wait_mc_seqno = wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0; wait_mc_seqno = std::max(wait_mc_seqno, last_known_masterchain_seqno_); diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index eaece0e87..1a9ca052e 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1294,6 +1294,9 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << " " << ton::create_shard_id(shard).to_str() << "\n"; } } + if (overlay->skip_public_msg_send_) { + td::TerminalIO::out() << "Don't send external messages to public overlays\n"; + } td::TerminalIO::out() << "\n"; } return td::Status::OK(); @@ -1853,6 +1856,9 @@ td::Status GetCollationManagerStatsQuery::receive(td::BufferSlice data) { } sb << td::StringBuilder::FixedDouble(collator->last_ping_ago_, 3) << ": " << status; } + if (collator->banned_for_ > 0.0) { + sb << " banned_for=" << td::StringBuilder::FixedDouble(std::max(collator->banned_for_, 0.0), 3); + } td::TerminalIO::out() << sb.as_cslice() << "\n"; } } diff --git a/validator-engine/BlockParserAsync.cpp b/validator-engine/BlockParserAsync.cpp index 1513f7bae..1e41c8c03 100644 --- a/validator-engine/BlockParserAsync.cpp +++ b/validator-engine/BlockParserAsync.cpp @@ -611,7 +611,7 @@ namespace ton::validator { LOG(DEBUG) << "Parse: " << blkid.to_str() << " ValueFlow success"; auto in_msg_dict = std::make_unique(vm::load_cell_slice_ref(extra.in_msg_descr), 256, - block::tlb::aug_InMsgDescr); + block::tlb::aug_InMsgDescrDefault); std::vector in_msgs_json; while (!in_msg_dict->is_empty()) { @@ -628,7 +628,7 @@ namespace ton::validator { LOG(DEBUG) << "Parse: " << blkid.to_str() << " in_msg_dict success"; auto out_msg_dict = std::make_unique(vm::load_cell_slice_ref(extra.out_msg_descr), 256, - block::tlb::aug_OutMsgDescr); + block::tlb::aug_OutMsgDescrDefault); std::vector out_msgs_json; while (!out_msg_dict->is_empty()) { diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index 94363c79c..6d4b00bd6 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -26,4 +26,4 @@ target_include_directories(validatorsession PUBLIC $/.. ${OPENSSL_INCLUDE_DIR} ) -target_link_libraries(validatorsession PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec overlay catchain) +target_link_libraries(validatorsession PRIVATE tdutils tdactor adnl rldp2 tl_api dht tdfec overlay catchain) diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index 2fc7d7a5f..9aa376e2a 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -18,6 +18,7 @@ #include "tl-utils/tl-utils.hpp" #include "vm/boc.h" #include "td/utils/lz4.h" +#include "vm/boc-compression.h" #include "validator-session-types.h" namespace ton::validatorsession { @@ -40,13 +41,35 @@ td::Result> deserialize_candi if (!compression_enabled) { return fetch_tl_object(data, true); } - TRY_RESULT(f, fetch_tl_object(data, true)); - if (f->decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_, proto_version)); - return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(p.first), - std::move(p.second)); + TRY_RESULT(f, fetch_tl_object(data, true)); + td::Result> res; + ton_api::downcast_call(*f, td::overloaded( + [&](ton_api::validatorSession_candidate& c) { + res = td::Status::Error("Received decompressed tl object, while compression_enabled=true"); + }, + [&](ton_api::validatorSession_compressedCandidate& c) { + res = [&]() -> td::Result> { + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, proto_version)); + return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), + std::move(p.second)); + }(); + }, + [&](ton_api::validatorSession_compressedCandidateV2& c) { + res = [&]() -> td::Result> { + if (c.data_.size() > max_decompressed_data_size) { + return td::Status::Error("Compressed data is too big"); + } + TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, + max_decompressed_data_size, proto_version)); + return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), + std::move(p.second)); + }(); + })); + return res; } td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, @@ -69,13 +92,20 @@ td::Result compress_candidate_data(td::Slice block, td::Slice c } td::Result> decompress_candidate_data(td::Slice compressed, + bool improved_compression, int decompressed_size, + int max_decompressed_size, int proto_version) { - TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); - if (decompressed.size() != (size_t)decompressed_size) { - return td::Status::Error("decompressed size mismatch"); + std::vector> roots; + if (!improved_compression) { + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + if (decompressed.size() != (size_t)decompressed_size) { + return td::Status::Error("decompressed size mismatch"); + } + TRY_RESULT_ASSIGN(roots, vm::std_boc_deserialize_multi(decompressed)); + } else { + TRY_RESULT_ASSIGN(roots, vm::boc_decompress(compressed, max_decompressed_size)); } - TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); if (roots.empty()) { return td::Status::Error("boc is empty"); } @@ -83,7 +113,7 @@ td::Result> decompress_candidate_dat roots.erase(roots.begin()); int collated_data_mode = proto_version >= 5 ? 2 : 31; TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), collated_data_mode)); - LOG(DEBUG) << "Decompressing block candidate: " << compressed.size() << " -> " + LOG(DEBUG) << "Decompressing block candidate " << (improved_compression ? "V2:" : ":") << compressed.size() << " -> " << block_data.size() + collated_data.size(); return std::make_pair(std::move(block_data), std::move(collated_data)); } diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index 7cc77f0be..9ab53cc89 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -30,7 +30,9 @@ td::Result> deserialize_candi td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, size_t& decompressed_size); td::Result> decompress_candidate_data(td::Slice compressed, + bool improved_compression, int decompressed_size, + int max_decompressed_size, int proto_version); } // namespace ton::validatorsession diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 020079739..982d2642b 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -230,7 +230,7 @@ struct NewValidatorGroupStats { std::vector prev; td::uint32 self_idx = 0; PublicKeyHash self = PublicKeyHash::zero(); - std::vector nodes; + std::vector nodes{}; tl_object_ptr tl() const { std::vector> prev_arr; @@ -257,7 +257,7 @@ struct EndValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); double timestamp = -1.0; PublicKeyHash self = PublicKeyHash::zero(); - std::vector nodes; + std::vector nodes{}; tl_object_ptr tl() const { std::vector> nodes_arr; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index e63b0e569..27c105929 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -18,7 +18,6 @@ */ #include "validator-session.hpp" #include "td/utils/Random.h" -#include "td/utils/crypto.h" #include "candidate-serializer.h" #include "td/utils/overloaded.h" #include "ton/ton-tl.hpp" @@ -63,7 +62,7 @@ void ValidatorSessionImpl::process_blocks(std::vector auto to_approve = real_state_->choose_blocks_to_approve(description(), local_idx()); for (auto &block : to_approve) { auto id = SentBlock::get_block_id(block); - if (approved_.count(id) && approved_[id].first <= td::Clocks::system()) { + if (approved_.contains(id) && approved_[id].first <= td::Clocks::system()) { msgs.emplace_back(create_tl_object( cur_round_, id, approved_[id].second.clone())); cnt++; @@ -223,6 +222,19 @@ bool ValidatorSessionImpl::ensure_candidate_unique(td::uint32 src_idx, td::uint3 void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, bool is_overlay_broadcast, bool is_startup) { + bool is_optimistic = false; + ValidatorSessionCandidateId optimistic_prev_candidate = ValidatorSessionCandidateId::zero(); + if (is_overlay_broadcast) { + auto R = fetch_tl_object(data, true); + if (R.is_ok()) { + if (src == local_id()) { + return; + } + is_optimistic = true; + optimistic_prev_candidate = R.ok_ref()->prev_candidate_id_; + data = std::move(R.ok_ref()->data_); + } + } // Note: src is not necessarily equal to the sender of this message: // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); @@ -262,13 +274,70 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } + if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || + block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK || (block_round == 0 && is_optimistic)) { + VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id + << "]: bad round=" << block_round << " cur_round" << cur_round_; + return; + } + + BroadcastInfo broadcast_info{.candidate_id = block_id, + .received_at = td::Clocks::system(), + .deserialize_time = deserialize_time, + .serialized_size = data.size(), + .file_hash = file_hash, + .collated_data_hash = collated_data_file_hash}; + + if (is_optimistic) { + if (optimistic_broadcasts_.contains({block_round, src_idx})) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: duplicate optimistic broadcast for round " << block_round; + return; + } + int priority = description().get_node_priority(src_idx, block_round); + if (priority < 0) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: node is not a producer in round " << block_round; + return; + } + if (block_round > cur_round_) { + OptimisticBroadcast &optimistic_broadcast = optimistic_broadcasts_[{block_round, src_idx}]; + optimistic_broadcast.candidate = std::move(candidate); + optimistic_broadcast.prev_candidate_id = optimistic_prev_candidate; + optimistic_broadcast.broadcast_info = broadcast_info; + VLOG(VALIDATOR_SESSION_WARNING) << this << ": received optimistic broadcast " << block_id << " from " << src + << ", round " << block_round; + validate_optimistic_broadcast( + BlockSourceInfo{description().get_source_public_key(src_idx), + BlockCandidatePriority{block_round, block_round, priority}}, + optimistic_broadcast.candidate->root_hash_, optimistic_broadcast.candidate->data_.clone(), + optimistic_broadcast.candidate->collated_data_.clone(), optimistic_broadcast.prev_candidate_id); + return; + } + if (SentBlock::get_block_id(real_state_->get_committed_block(description(), block_round - 1)) != + optimistic_prev_candidate) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: dropping optimistic broadcast for round " << block_round + << " - prev candidate mismatch"; + return; + } + } + process_received_block(block_round, src, src_idx, std::move(candidate), broadcast_info, is_overlay_broadcast, + is_startup); +} + +void ValidatorSessionImpl::process_received_block(td::uint32 block_round, PublicKeyHash src, td::uint32 src_idx, + tl_object_ptr candidate, + const BroadcastInfo &info, bool is_overlay_broadcast, + bool is_startup) { + ValidatorSessionCandidateId block_id = info.candidate_id; auto stat = stats_get_candidate_stat(block_round, src, block_id); if (stat) { if (stat->block_status == ValidatorSessionStats::status_none) { stat->block_status = ValidatorSessionStats::status_received; } if (stat->got_block_at <= 0.0) { - stat->got_block_at = td::Clocks::system(); + stat->got_block_at = info.received_at; if (is_overlay_broadcast) { stat->got_block_by = ValidatorSessionStats::recv_broadcast; } else if (is_startup) { @@ -277,19 +346,13 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice stat->got_block_by = ValidatorSessionStats::recv_query; } } - stat->deserialize_time = deserialize_time; - stat->serialized_size = data.size(); + stat->deserialize_time = info.deserialize_time; + stat->serialized_size = (int)info.serialized_size; stat->block_id.root_hash = candidate->root_hash_; - stat->block_id.file_hash = file_hash; - stat->collated_data_hash = collated_data_file_hash; + stat->block_id.file_hash = info.file_hash; + stat->collated_data_hash = info.collated_data_hash; } - if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || - block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK) { - VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id - << "]: bad round=" << block_round << " cur_round" << cur_round_; - return; - } auto it = blocks_.find(block_id); if (it != blocks_.end()) { it->second->round_ = std::max(it->second->round_, block_round); @@ -309,16 +372,22 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice } blocks_[block_id] = std::move(candidate); + if (auto it = block_waiters_.find(block_id); it != block_waiters_.end()) { + for (auto &promise : it->second) { + promise.set_result(td::Unit()); + } + block_waiters_.erase(it); + } VLOG(VALIDATOR_SESSION_WARNING) << this << ": received broadcast " << block_id; if (block_round != cur_round_) { return; } - CHECK(!pending_approve_.count(block_id)); - CHECK(!approved_.count(block_id)); - CHECK(!pending_reject_.count(block_id)); - CHECK(!rejected_.count(block_id)); + CHECK(!pending_approve_.contains(block_id)); + CHECK(!approved_.contains(block_id)); + CHECK(!pending_reject_.contains(block_id)); + CHECK(!rejected_.contains(block_id)); auto v = virtual_state_->choose_blocks_to_approve(description(), local_idx()); for (auto &b : v) { @@ -329,6 +398,37 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice } } +void ValidatorSessionImpl::validate_optimistic_broadcast(BlockSourceInfo source_info, + ValidatorSessionRootHash root_hash, td::BufferSlice data, + td::BufferSlice collated_data, + ValidatorSessionCandidateId prev_candidate_id) { + if (source_info.priority.round <= cur_round_) { + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id() << " : too old"; + return; + } + auto it = blocks_.find(prev_candidate_id); + if (it == blocks_.end()) { + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id() << " : wait for prev block"; + block_waiters_[prev_candidate_id].push_back( + [=, SelfId = actor_id(this), data = std::move(data), + collated_data = std::move(collated_data)](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::validate_optimistic_broadcast, source_info, + root_hash, std::move(data), std::move(collated_data), prev_candidate_id); + } + }); + return; + } + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id(); + callback_->on_optimistic_candidate( + source_info, root_hash, std::move(data), std::move(collated_data), + description().get_source_public_key(description().get_source_idx(PublicKeyHash{it->second->src_})), + it->second->root_hash_, it->second->data_.clone(), it->second->collated_data_.clone()); +} + void ValidatorSessionImpl::process_message(PublicKeyHash src, td::BufferSlice data) { } @@ -421,7 +521,7 @@ void ValidatorSessionImpl::candidate_decision_ok(td::uint32 round, ValidatorSess stat->block_status = ValidatorSessionStats::status_approved; stat->comment = PSTRING() << "ts=" << ok_from; stat->validation_time = validation_time; - stat->gen_utime = (double)ok_from; + stat->gen_utime = (int)ok_from; stat->validated_at = td::Clocks::system(); stat->validation_cached = validation_cached; } @@ -476,7 +576,11 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate stat->block_status = ValidatorSessionStats::status_received; stat->collation_time = collation_time; stat->collated_at = td::Clocks::system(); - stat->got_block_at = td::Clocks::system(); + if (auto it = sent_candidates_.find(block_id); it != sent_candidates_.end()) { + stat->got_block_at = it->second.sent_at; + } else { + stat->got_block_at = td::Clocks::system(); + } stat->got_block_by = ValidatorSessionStats::recv_collated; stat->collation_cached = c.is_cached; stat->self_collated = c.self_collated; @@ -487,19 +591,21 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate if (round != cur_round_) { return; } - td::Timer serialize_timer; - auto b = create_tl_object(local_id().tl(), round, c.candidate.id.root_hash, - std::move(c.candidate.data), - std::move(c.candidate.collated_data)); - auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok(); + SentCandidateStats &send_stats = send_candidate_broadcast(round, c.candidate); if (stat) { - stat->serialize_time = serialize_timer.elapsed(); - stat->serialized_size = B.size(); + stat->serialize_time = send_stats.serialize_time; + stat->serialized_size = send_stats.serialized_size; } - td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(B)); - - blocks_.emplace(block_id, std::move(b)); + blocks_[block_id] = create_tl_object( + local_id().tl(), round, c.candidate.id.root_hash, std::move(c.candidate.data), + std::move(c.candidate.collated_data)); + if (auto it = block_waiters_.find(block_id); it != block_waiters_.end()) { + for (auto &promise : it->second) { + promise.set_result(td::Unit()); + } + block_waiters_.erase(it); + } pending_generate_ = false; generated_ = true; generated_block_ = block_id; @@ -507,6 +613,31 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate request_new_block(true); } +ValidatorSessionImpl::SentCandidateStats &ValidatorSessionImpl::send_candidate_broadcast( + td::uint32 round, const BlockCandidate &candidate, + td::optional optimistic_prev_candidate) { + ValidatorSessionCandidateId candidate_id = description().candidate_id( + local_idx(), candidate.id.root_hash, candidate.id.file_hash, candidate.collated_file_hash); + SentCandidateStats &stats = sent_candidates_[candidate_id]; + if (stats.sent) { + return stats; + } + stats.sent = true; + td::Timer serialize_timer; + auto b = create_tl_object( + local_id().tl(), round, candidate.id.root_hash, candidate.data.clone(), candidate.collated_data.clone()); + auto data = serialize_candidate(b, compress_block_candidates_).move_as_ok(); + stats.serialize_time = serialize_timer.elapsed(); + stats.sent_at = td::Clocks::system(); + stats.serialized_size = data.size(); + if (optimistic_prev_candidate) { + data = create_serialize_tl_object( + 0, optimistic_prev_candidate.value(), std::move(data)); + } + td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(data)); + return stats; +} + void ValidatorSessionImpl::signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature) { if (round != cur_round_) { return; @@ -586,7 +717,7 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { return; } } - if (pending_approve_.count(block_id) || rejected_.count(block_id)) { + if (pending_approve_.contains(block_id) || rejected_.contains(block_id)) { return; } @@ -651,7 +782,7 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { description().get_node_priority(block->get_src_idx(), cur_round_)}}, B->root_hash_, B->data_.clone(), B->collated_data_.clone(), std::move(P)); } else if (T.is_in_past()) { - if (!active_requests_.count(block_id)) { + if (!active_requests_.contains(block_id)) { auto v = virtual_state_->get_block_approvers(description(), block_id); if (v.size() > 0) { auto id = description().get_source_id(v[td::Random::fast(0, static_cast(v.size() - 1))]); @@ -665,7 +796,6 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { VLOG(VALIDATOR_SESSION_WARNING) << print_id << ": failed to get candidate " << hash << " from " << id << ": " << R.move_as_error(); } else { - LOG(ERROR) << "QQQQQ Got block " << R.ok().size(); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok(), candidate_id, false, false); } @@ -796,9 +926,9 @@ void ValidatorSessionImpl::check_all() { for (auto &B : to_approve) { if (B) { auto block_id = SentBlock::get_block_id(B); - auto pending = pending_approve_.count(block_id) == 1; - auto rejected = rejected_.count(block_id) == 1; - auto accepted = approved_.count(block_id) == 1; + auto pending = pending_approve_.contains(block_id); + auto rejected = rejected_.contains(block_id); + auto accepted = approved_.contains(block_id); sb << " " << block_id << " pending: " << pending << " rejected: " << rejected << " accepted: " << accepted << "\n"; } else { @@ -960,6 +1090,18 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { round_started_at_ = td::Timestamp::now(); round_debug_at_ = td::Timestamp::in(60.0); + for (auto it = optimistic_broadcasts_.begin(); it != optimistic_broadcasts_.end() && it->first.first <= cur_round_;) { + OptimisticBroadcast &optimistic_broadcast = it->second; + auto [block_round, src_idx] = it->first; + if (SentBlock::get_block_id(real_state_->get_committed_block(description(), block_round - 1)) == + optimistic_broadcast.prev_candidate_id) { + process_received_block(block_round, description().get_source_id(src_idx), src_idx, + std::move(optimistic_broadcast.candidate), optimistic_broadcast.broadcast_info, true, + false); + } + it = optimistic_broadcasts_.erase(it); + } + check_all(); } @@ -997,7 +1139,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) : unique_hash_(session_id) @@ -1010,6 +1152,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i , overlay_manager_(overlays) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { compress_block_candidates_ = opts.proto_version >= 4; + allow_optimistic_generation_ = opts.proto_version >= 6; description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id); src_round_candidate_.resize(description_->get_total_nodes()); } @@ -1120,7 +1263,6 @@ void ValidatorSessionImpl::start_up() { virtual_state_ = real_state_; check_all(); - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, description().get_source_adnl_id(local_idx())); } void ValidatorSessionImpl::stats_init() { @@ -1200,7 +1342,7 @@ ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat( auto it2 = stats_pending_approve_.find({round, it->candidate_id}); if (it2 != stats_pending_approve_.end()) { for (td::uint32 node_id : it2->second) { - it->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + process_approve(node_id, round, it->candidate_id); } stats_pending_approve_.erase(it2); } @@ -1249,13 +1391,7 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val if (obj.candidate_ == skip_round_candidate_id()) { return; } - auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); - if (stat) { - stat->set_approved_by(node_id, description().get_node_weight(node_id), - description().get_total_weight()); - } else { - stats_pending_approve_[{obj.round_, obj.candidate_}].push_back(node_id); - } + process_approve(node_id, obj.round_, obj.candidate_); }, [&](const ton_api::validatorSession_message_commit &obj) { if (obj.candidate_ == skip_round_candidate_id()) { @@ -1272,11 +1408,75 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val [](const auto &) {})); } +void ValidatorSessionImpl::process_approve(td::uint32 node_id, td::uint32 round, + ValidatorSessionCandidateId candidate_id) { + auto stat = stats_get_candidate_stat_by_id(round, candidate_id); + if (!stat) { + stats_pending_approve_[{round, candidate_id}].push_back(node_id); + return; + } + + bool was_approved_66pct = stat->approved_66pct_at > 0.0; + stat->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + bool is_approved_66pct = stat->approved_66pct_at > 0.0; + + if (allow_optimistic_generation_ && !was_approved_66pct && is_approved_66pct && cur_round_ == round && + cur_round_ == first_block_round_ && description().get_node_priority(local_idx(), round + 1) == 0 && + blocks_.contains(candidate_id) && + description().get_node_priority(description().get_source_idx(stat->validator_id), cur_round_) == 0) { + if (blocks_.contains(candidate_id)) { + generate_block_optimistic(round, candidate_id); + } else { + block_waiters_[candidate_id].push_back([=, SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::generate_block_optimistic, round, candidate_id); + } + }); + } + } +} + +void ValidatorSessionImpl::generate_block_optimistic(td::uint32 cur_round, + ValidatorSessionCandidateId prev_candidate_id) { + if (cur_round != cur_round_) { + return; + } + auto it = blocks_.find(prev_candidate_id); + if (it == blocks_.end()) { + return; + } + auto &block = it->second; + auto stat = stats_get_candidate_stat_by_id(cur_round, prev_candidate_id); + if (!stat) { + return; + } + callback_->generate_block_optimistic(BlockSourceInfo{description().get_source_public_key(local_idx()), + BlockCandidatePriority{cur_round + 1, cur_round + 1, 0}}, + block->data_.clone(), block->root_hash_, stat->block_id.file_hash, + [=, SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Optimistic generation error: " << R.move_as_error(); + return; + } + td::actor::send_closure(SelfId, + &ValidatorSessionImpl::generated_optimistic_candidate, + cur_round + 1, R.move_as_ok(), prev_candidate_id); + }); +} + +void ValidatorSessionImpl::generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, + ValidatorSessionCandidateId prev_candidate) { + if (cur_round_ > round) { + return; + } + send_candidate_broadcast(round, candidate.candidate, prev_candidate); +} + td::actor::ActorOwn ValidatorSession::create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) { return td::actor::create_actor("session", session_id, std::move(opts), local_id, std::move(nodes), std::move(callback), keyring, adnl, rldp, diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 641fb4866..f8e2d62e7 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -20,7 +20,7 @@ #include "adnl/adnl.h" #include "adnl/utils.hpp" -#include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "ton/ton-types.h" @@ -93,6 +93,15 @@ class ValidatorSession : public td::actor::Actor { ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void generate_block_optimistic(BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, + td::Promise promise) { + } + virtual void on_optimistic_candidate(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, + td::BufferSlice data, td::BufferSlice collated_data, PublicKey prev_source, + ValidatorSessionRootHash prev_root_hash, td::BufferSlice prev_data, + td::BufferSlice prev_collated_data) { + } virtual ~Callback() = default; }; @@ -109,7 +118,7 @@ class ValidatorSession : public td::actor::Actor { catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); virtual ~ValidatorSession() = default; }; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 04fdc875c..4861c52e1 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -76,6 +76,7 @@ class ValidatorSessionImpl : public ValidatorSession { std::map> blocks_; // src_round_candidate_[src_id][round] -> candidate id std::vector> src_round_candidate_; + std::map>> block_waiters_; catchain::CatChainSessionId unique_hash_; @@ -85,7 +86,7 @@ class ValidatorSessionImpl : public ValidatorSession { td::actor::ActorId keyring_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId overlay_manager_; td::actor::ActorOwn catchain_; std::unique_ptr description_; @@ -161,6 +162,7 @@ class ValidatorSessionImpl : public ValidatorSession { bool catchain_started_ = false; bool allow_unsafe_self_blocks_resync_; bool compress_block_candidates_ = false; + bool allow_optimistic_generation_ = false; ValidatorSessionStats cur_stats_; bool stats_inited_ = false; @@ -168,6 +170,30 @@ class ValidatorSessionImpl : public ValidatorSession { stats_pending_approve_; // round, candidate_id -> approvers std::map, std::vector> stats_pending_sign_; // round, candidate_id -> signers + + struct SentCandidateStats { + bool sent = false; + double sent_at = -1.0; + double serialize_time = -1.0; + size_t serialized_size = 0; + }; + std::map sent_candidates_; + + struct BroadcastInfo { + ValidatorSessionCandidateId candidate_id; + double received_at = -1.0; + double deserialize_time = -1.0; + size_t serialized_size = 0; + ValidatorSessionFileHash file_hash; + ValidatorSessionCollatedDataFileHash collated_data_hash; + }; + struct OptimisticBroadcast { + tl_object_ptr candidate; + ValidatorSessionCandidateId prev_candidate_id; + BroadcastInfo broadcast_info; + }; + std::map, OptimisticBroadcast> optimistic_broadcasts_; // round, src -> broadcast + void stats_init(); void stats_add_round(); ValidatorSessionStats::Producer *stats_get_candidate_stat( @@ -176,12 +202,16 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionStats::Producer *stats_get_candidate_stat_by_id(td::uint32 round, ValidatorSessionCandidateId candidate_id); void stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action); + void process_approve(td::uint32 node_id, td::uint32 round, ValidatorSessionCandidateId candidate_id); + void generate_block_optimistic(td::uint32 cur_round, ValidatorSessionCandidateId prev_candidate_id); + void generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, + ValidatorSessionCandidateId prev_candidate); public: ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); void start_up() override; void alarm() override; @@ -205,6 +235,12 @@ class ValidatorSessionImpl : public ValidatorSession { bool ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, ValidatorSessionCandidateId block_id); void process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, bool is_overlay_broadcast, bool is_startup); + void process_received_block(td::uint32 block_round, PublicKeyHash src, td::uint32 src_idx, + tl_object_ptr candidate, const BroadcastInfo &info, + bool is_overlay_broadcast, bool is_startup); + void validate_optimistic_broadcast(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, + td::BufferSlice data, td::BufferSlice collated_data, + ValidatorSessionCandidateId prev_candidate_id); void process_message(PublicKeyHash src, td::BufferSlice data); void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); @@ -218,6 +254,9 @@ class ValidatorSessionImpl : public ValidatorSession { td::BufferSlice signature); void generated_block(td::uint32 round, GeneratedCandidate c, double collation_time); + SentCandidateStats &send_candidate_broadcast( + td::uint32 round, const BlockCandidate &candidate, + td::optional optimistic_prev_candidate = {}); void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature); void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index e3819e32b..0aeef4f9f 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -78,7 +78,9 @@ set(VALIDATOR_HEADERS shard-block-retainer.hpp collation-manager.hpp - collator-node.hpp + collator-node/collator-node.hpp + collator-node/collator-node-session.hpp + collator-node/utils.hpp manager-disk.h manager-disk.hpp manager-init.h @@ -95,7 +97,9 @@ set(VALIDATOR_SOURCE apply-block.cpp block-handle.cpp collation-manager.cpp - collator-node.cpp + collator-node/collator-node.cpp + collator-node/collator-node-session.cpp + collator-node/utils.cpp get-next-key-blocks.cpp import-db-slice.cpp import-db-slice-local.cpp diff --git a/validator/apply-block.cpp b/validator/apply-block.cpp index d6094ff8b..8c45f851e 100644 --- a/validator/apply-block.cpp +++ b/validator/apply-block.cpp @@ -166,7 +166,7 @@ void ApplyBlock::written_block_data() { }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, apply_block_priority(), timeout_, - std::move(P)); + true, std::move(P)); } } diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 853ac37ae..7d694af98 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -16,7 +16,8 @@ */ #include "collation-manager.hpp" -#include "collator-node.hpp" +#include "collator-node/collator-node.hpp" +#include "collator-node/utils.hpp" #include "fabric.h" #include "td/utils/Random.h" @@ -29,6 +30,29 @@ namespace ton::validator { void CollationManager::start_up() { td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); update_collators_list(*opts_->get_collators_list()); + + class Cb : public adnl::Adnl::Callback { + public: + explicit Cb(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &CollationManager::receive_query, src, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_requestBlockCallback::ID), + std::make_unique(actor_id(this))); +} + +void CollationManager::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_requestBlockCallback::ID)); } void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, @@ -38,11 +62,15 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha td::Promise promise, int proto_version) { if (shard.is_masterchain()) { run_collate_query( - shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) { + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = std::move(prev), + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options()}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; - }), - adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0); + })); return; } collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), @@ -50,12 +78,55 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha proto_version); } +void CollationManager::collate_block_optimistic(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + BlockIdExt prev_block_id, td::BufferSlice prev_block, + Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, + td::Promise promise, int proto_version) { + if (shard.is_masterchain()) { + TRY_RESULT_PROMISE(promise, prev_block_data, create_block(prev_block_id, std::move(prev_block))); + run_collate_query( + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = {prev_block_id}, + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options(), + .optimistic_prev_block = std::move(prev_block_data)}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + })); + return; + } + + auto& entry = optimistic_prev_cache_[prev_block_id]; + entry.block_data = std::move(prev_block); + ++entry.refcnt; + promise = [this, SelfId = actor_id(this), prev_block_id, + promise = std::move(promise)](td::Result R) mutable { + promise.set_result(std::move(R)); + td::actor::send_lambda_later(SelfId, [=, this]() { + auto it = optimistic_prev_cache_.find(prev_block_id); + CHECK(it != optimistic_prev_cache_.end()); + CHECK(it->second.refcnt > 0); + if (--it->second.refcnt == 0) { + optimistic_prev_cache_.erase(it); + } + }); + }; + + collate_shard_block(shard, min_masterchain_block_id, {prev_block_id}, creator, priority, std::move(validator_set), + max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0), + proto_version, true); +} + void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, td::Timestamp timeout, - int proto_version) { + int proto_version, bool is_optimistic) { TRY_STATUS_PROMISE(promise, cancellation_token.check()); ShardInfo* s = select_shard_info(shard); if (s == nullptr) { @@ -66,54 +137,83 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas adnl::AdnlNodeIdShort selected_collator = adnl::AdnlNodeIdShort::zero(); size_t selected_idx = 0; - switch (s->select_mode) { - case CollatorsList::mode_random: { - int cnt = 0; - for (size_t i = 0; i < s->collators.size(); ++i) { - adnl::AdnlNodeIdShort collator = s->collators[i]; - if (collators_[collator].alive) { - ++cnt; - if (td::Random::fast(1, cnt) == 1) { + for (int allow_banned = 0; allow_banned < 2; ++allow_banned) { + auto check_collator = [&](const adnl::AdnlNodeIdShort& id) -> bool { + auto& collator = collators_[id]; + if (!collator.alive) { + return false; + } + if (collator.banned_until && !allow_banned) { + return false; + } + if (is_optimistic && collator.version < CollatorNode::VERSION_OPTIMISTIC_COLLATE) { + return false; + } + return true; + }; + switch (s->select_mode) { + case CollatorsList::mode_random: { + int cnt = 0; + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (check_collator(collator)) { + ++cnt; + if (td::Random::fast(1, cnt) == 1) { + selected_collator = collator; + selected_idx = i; + } + } + } + break; + } + case CollatorsList::mode_ordered: { + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (check_collator(collator)) { selected_collator = collator; selected_idx = i; + break; } } + break; } - break; - } - case CollatorsList::mode_ordered: { - for (size_t i = 0; i < s->collators.size(); ++i) { - adnl::AdnlNodeIdShort collator = s->collators[i]; - if (collators_[collator].alive) { - selected_collator = collator; - selected_idx = i; - break; + case CollatorsList::mode_round_robin: { + size_t iters = 0; + for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) { + adnl::AdnlNodeIdShort& collator = s->collators[i]; + if (check_collator(collator)) { + selected_collator = collator; + selected_idx = i; + s->cur_idx = (i + 1) % s->collators.size(); + break; + } } + break; } - break; } - case CollatorsList::mode_round_robin: { - size_t iters = 0; - for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) { - adnl::AdnlNodeIdShort& collator = s->collators[i]; - if (collators_[collator].alive) { - selected_collator = collator; - selected_idx = i; - s->cur_idx = (i + 1) % s->collators.size(); - break; - } - } + if (!selected_collator.is_zero() || s->self_collate) { break; } } if (selected_collator.is_zero() && s->self_collate) { + td::Ref optimistic_prev_block; + if (is_optimistic) { + CHECK(prev.size() == 1); + TRY_RESULT_PROMISE_ASSIGN(promise, optimistic_prev_block, + create_block(prev[0], optimistic_prev_cache_.at(prev[0]).block_data.clone())); + } run_collate_query( - shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) { + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = std::move(prev), + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options(), + .optimistic_prev_block = std::move(optimistic_prev_block)}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; - }), - adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0); + })); return; } @@ -146,19 +246,26 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas [=, promise = std::move(promise)]() mutable { td::actor::send_closure(SelfId, &CollationManager::collate_shard_block, shard, min_masterchain_block_id, prev, creator, priority, validator_set, max_answer_size, cancellation_token, - std::move(promise), timeout, proto_version); + std::move(promise), timeout, proto_version, is_optimistic); }, retry_at); }; if (selected_collator.is_zero()) { - P.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node")); + P.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no suitable collator node")); return; } - td::BufferSlice query = create_serialize_tl_object( - create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), - priority.round, priority.first_block_round, priority.priority); + td::BufferSlice query; + if (is_optimistic) { + query = create_serialize_tl_object( + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), + priority.round, priority.first_block_round, priority.priority); + } else { + query = create_serialize_tl_object( + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), + priority.round, priority.first_block_round, priority.priority); + } LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "(" << selected_collator << ")"; @@ -172,9 +279,8 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas return; } TRY_RESULT_PROMISE(P, f, fetch_tl_object(data, true)); - TRY_RESULT_PROMISE( - P, candidate, - CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size), proto_version)); + TRY_RESULT_PROMISE(P, candidate, + deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size), proto_version)); if (candidate.pubkey.as_bits256() != creator.as_bits256()) { P.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; @@ -274,11 +380,21 @@ void CollationManager::get_stats( } obj->last_ping_ago_ = collator.last_ping_at ? td::Time::now() - collator.last_ping_at.at() : -1.0; obj->last_ping_status_ = collator.last_ping_status.is_ok() ? "OK" : collator.last_ping_status.message().str(); + obj->banned_for_ = collator.banned_until ? collator.banned_until.in() : -1.0; stats->collators_.push_back(std::move(obj)); } promise.set_value(std::move(stats)); } +void CollationManager::ban_collator(adnl::AdnlNodeIdShort collator_id, std::string reason) { + auto it = collators_.find(collator_id); + if (it == collators_.end()) { + return; + } + alarm_timestamp().relax(it->second.banned_until = td::Timestamp::in(BAN_DURATION)); + LOG(ERROR) << "Ban collator " << collator_id << " for " << BAN_DURATION << "s: " << reason; +} + void CollationManager::update_collators_list(const CollatorsList& collators_list) { shards_.clear(); for (auto& [_, collator] : collators_) { @@ -329,12 +445,21 @@ CollationManager::ShardInfo* CollationManager::select_shard_info(ShardIdFull sha void CollationManager::alarm() { alarm_timestamp() = td::Timestamp::never(); for (auto& [id, collator] : collators_) { + if (collator.banned_until) { + if (collator.banned_until.is_in_past()) { + collator.banned_until = {}; + LOG(ERROR) << "Unban collator " << id; + } else { + alarm_timestamp().relax(collator.banned_until); + } + } if (collator.active_cnt == 0 || collator.sent_ping) { continue; } if (collator.ping_at.is_in_past()) { collator.sent_ping = true; - td::BufferSlice query = create_serialize_tl_object(0); + td::BufferSlice query = + create_serialize_tl_object(ton_api::collatorNode_pong::VERSION_MASK); td::Promise P = [=, id = id, SelfId = actor_id(this)](td::Result R) mutable { td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); }; @@ -370,9 +495,11 @@ void CollationManager::got_pong(adnl::AdnlNodeIdShort id, td::Resultflags_ & ton_api::collatorNode_pong::VERSION_MASK ? pong->version_ : 0; + LOG(DEBUG) << "pong from " << id << " : OK, version=" << collator.version; } collator.ping_at = td::Timestamp::in(td::Random::fast(10.0, 20.0)); if (collator.active_cnt && !collator.sent_ping) { @@ -392,4 +519,26 @@ void CollationManager::on_collate_query_error(adnl::AdnlNodeIdShort id) { } } +void CollationManager::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, + td::Promise promise) { + if (!collators_.contains(src)) { + promise.set_error(td::Status::Error("got request from unknown collator")); + return; + } + TRY_RESULT_PROMISE(promise, query, fetch_tl_object(data, true)); + BlockIdExt block_id = create_block_id(query->block_id_); + auto it = optimistic_prev_cache_.find(block_id); + if (it == optimistic_prev_cache_.end()) { + LOG(INFO) << "collatorNode.requestBlockCallback from " << src << " block " << block_id.to_str() << " : not found"; + promise.set_error(td::Status::Error("block not found")); + return; + } + LOG(INFO) << "collatorNode.requestBlockCallback from " << src << " block " << block_id.to_str() << " : OK"; + promise.set_value( + serialize_tl_object(serialize_candidate(BlockCandidate(Ed25519_PublicKey{td::Bits256::zero()}, block_id, + td::Bits256::zero(), it->second.block_data.clone(), {}), + true), + true)); +} + } // namespace ton::validator diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 5b8d2b7a0..6691a387e 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -27,11 +27,13 @@ class ValidatorManager; class CollationManager : public td::actor::Actor { public: CollationManager(adnl::AdnlNodeIdShort local_id, td::Ref opts, - td::actor::ActorId manager, td::actor::ActorId rldp) - : local_id_(local_id), opts_(opts), manager_(manager), rldp_(rldp) { + td::actor::ActorId manager, td::actor::ActorId adnl, + td::actor::ActorId rldp) + : local_id_(local_id), opts_(opts), manager_(manager), adnl_(adnl), rldp_(rldp) { } void start_up() override; + void tear_down() override; void alarm() override; void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, @@ -39,6 +41,12 @@ class CollationManager : public td::actor::Actor { td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, int proto_version); + void collate_block_optimistic(ShardIdFull shard, BlockIdExt min_masterchain_block_id, BlockIdExt prev_block_id, + td::BufferSlice prev_block, Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise, + int proto_version); + void update_options(td::Ref opts); void validator_group_started(ShardIdFull shard); @@ -46,17 +54,20 @@ class CollationManager : public td::actor::Actor { void get_stats(td::Promise> promise); + void ban_collator(adnl::AdnlNodeIdShort collator_id, std::string reason); + private: adnl::AdnlNodeIdShort local_id_; td::Ref opts_; td::actor::ActorId manager_; + td::actor::ActorId adnl_; td::actor::ActorId rldp_; void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, - td::Timestamp timeout, int proto_version); + td::Timestamp timeout, int proto_version, bool is_optimistic = false); void update_collators_list(const CollatorsList& collators_list); @@ -67,6 +78,9 @@ class CollationManager : public td::actor::Actor { size_t active_cnt = 0; td::Timestamp last_ping_at = td::Timestamp::never(); td::Status last_ping_status = td::Status::Error("not pinged"); + int version = -1; + // Collator is banned when in returns invalid block + td::Timestamp banned_until = td::Timestamp::never(); }; std::map collators_; @@ -86,6 +100,16 @@ class CollationManager : public td::actor::Actor { ShardInfo* select_shard_info(ShardIdFull shard); void got_pong(adnl::AdnlNodeIdShort id, td::Result R); void on_collate_query_error(adnl::AdnlNodeIdShort id); + + void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + + struct OptimisticPrevCache { + td::BufferSlice block_data; + size_t refcnt = 0; + }; + std::map optimistic_prev_cache_; + + static constexpr double BAN_DURATION = 300.0; }; } // namespace ton::validator diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp deleted file mode 100644 index 4b0d7e140..000000000 --- a/validator/collator-node.cpp +++ /dev/null @@ -1,681 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#include "collator-node.hpp" -#include "ton/ton-tl.hpp" -#include "fabric.h" -#include "block-auto.h" -#include "block-db.h" -#include "td/utils/lz4.h" -#include "checksum.h" -#include "impl/collator-impl.h" -#include "impl/shard.hpp" -#include "validator-session/candidate-serializer.h" - -namespace ton::validator { - -CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::Ref opts, - td::actor::ActorId manager, td::actor::ActorId adnl, - td::actor::ActorId rldp) - : local_id_(local_id) - , opts_(std::move(opts)) - , manager_(std::move(manager)) - , adnl_(std::move(adnl)) - , rldp_(std::move(rldp)) { -} - -void CollatorNode::start_up() { - class Cb : public adnl::Adnl::Callback { - public: - explicit Cb(td::actor::ActorId id) : id_(std::move(id)) { - } - void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { - } - void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(id_, &CollatorNode::receive_query, src, std::move(data), std::move(promise)); - } - - private: - td::actor::ActorId id_; - }; - td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, - adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID), - std::make_unique(actor_id(this))); - td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, - adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID), - std::make_unique(actor_id(this))); - td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); -} - -void CollatorNode::tear_down() { - td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, - adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID)); - td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, - adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID)); -} - -void CollatorNode::add_shard(ShardIdFull shard) { - CHECK(shard.is_valid_ext() && !shard.is_masterchain()); - if (std::find(collating_shards_.begin(), collating_shards_.end(), shard) != collating_shards_.end()) { - return; - } - LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); - collating_shards_.push_back(shard); -} - -void CollatorNode::del_shard(ShardIdFull shard) { - auto it = std::find(collating_shards_.begin(), collating_shards_.end(), shard); - if (it != collating_shards_.end()) { - collating_shards_.erase(it); - } -} - -void CollatorNode::new_masterchain_block_notification(td::Ref state) { - last_masterchain_state_ = state; - - if (state->last_key_block_id().seqno() != last_key_block_seqno_) { - last_key_block_seqno_ = state->last_key_block_id().seqno(); - mc_config_status_ = check_mc_config(); - if (mc_config_status_.is_error()) { - LOG(ERROR) << "Cannot validate masterchain config (possibly outdated software):" << mc_config_status_; - } - } - - if (validator_adnl_ids_.empty() || state->is_key_state()) { - validator_adnl_ids_.clear(); - for (int next : {-1, 0, 1}) { - td::Ref vals = state->get_total_validator_set(next); - if (vals.not_null()) { - for (const ValidatorDescr& descr : vals->export_vector()) { - if (descr.addr.is_zero()) { - validator_adnl_ids_.insert( - adnl::AdnlNodeIdShort(PublicKey(pubkeys::Ed25519{descr.key.as_bits256()}).compute_short_id())); - } else { - validator_adnl_ids_.insert(adnl::AdnlNodeIdShort(descr.addr)); - } - } - } - } - } - - std::map> new_shards; - for (auto& v : state->get_shards()) { - auto shard = v->shard(); - if (v->before_split()) { - CHECK(!v->before_merge()); - new_shards.emplace(shard_child(shard, true), std::vector{v->top_block_id()}); - new_shards.emplace(shard_child(shard, false), std::vector{v->top_block_id()}); - } else if (v->before_merge()) { - ShardIdFull p_shard = shard_parent(shard); - auto it = new_shards.find(p_shard); - if (it == new_shards.end()) { - new_shards.emplace(p_shard, std::vector(2)); - } - bool left = shard_child(p_shard.shard, true) == shard.shard; - new_shards[p_shard][left ? 0 : 1] = v->top_block_id(); - } else { - new_shards.emplace(shard, std::vector{v->top_block_id()}); - } - } - - for (auto& [shard, prev] : new_shards) { - CatchainSeqno cc_seqno = state->get_validator_set(shard)->get_catchain_seqno(); - auto it = validator_groups_.emplace(shard, ValidatorGroupInfo{}); - ValidatorGroupInfo& info = it.first->second; - if (it.second || info.cc_seqno != cc_seqno) { - info.cleanup(); - info.cc_seqno = cc_seqno; - } - } - for (auto it = validator_groups_.begin(); it != validator_groups_.end();) { - if (new_shards.contains(it->first)) { - ++it; - } else { - it->second.cleanup(); - it = validator_groups_.erase(it); - } - } - for (auto& [shard, prev] : new_shards) { - ValidatorGroupInfo& info = validator_groups_[shard]; - update_validator_group_info(shard, std::move(prev), info.cc_seqno); - auto it = future_validator_groups_.find({shard, info.cc_seqno}); - if (it != future_validator_groups_.end()) { - for (auto& new_prev : it->second.pending_blocks) { - update_validator_group_info(shard, std::move(new_prev), info.cc_seqno); - } - for (auto& promise : it->second.promises) { - promise.set_value(td::Unit()); - } - future_validator_groups_.erase(it); - } - } - - for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end();) { - if (get_future_validator_group(it->first.first, it->first.second).is_ok()) { - ++it; - } else { - auto& future_group = it->second; - for (auto& promise : future_group.promises) { - promise.set_error(td::Status::Error("validator group is outdated")); - } - it = future_validator_groups_.erase(it); - } - } -} - -void CollatorNode::update_shard_client_handle(BlockHandle shard_client_handle) { - shard_client_handle_ = shard_client_handle; -} - -void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector prev, - CatchainSeqno cc_seqno) { - if (!can_collate_shard(shard)) { - return; - } - CHECK(prev.size() == 1 || prev.size() == 2); - BlockSeqno next_block_seqno = prev[0].seqno() + 1; - if (prev.size() == 2) { - next_block_seqno = std::max(next_block_seqno, prev[1].seqno() + 1); - } - auto it = validator_groups_.find(shard); - if (it != validator_groups_.end()) { - ValidatorGroupInfo& info = it->second; - if (info.cc_seqno == cc_seqno) { // block from currently known validator group - if (info.next_block_seqno < next_block_seqno) { - LOG(DEBUG) << "updated validator group info: shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << next_block_seqno; - info.next_block_seqno = next_block_seqno; - info.prev = std::move(prev); - for (auto cache_it = info.cache.begin(); cache_it != info.cache.end();) { - auto& [cached_prev, cache_entry] = *cache_it; - if (cache_entry->block_seqno < info.next_block_seqno) { - cache_entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << cache_entry->block_seqno - << " is too small, expected " << info.next_block_seqno)); - if (!cache_entry->has_external_query_at && cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": nobody asked for block, but we tried to generate it"; - } - if (cache_entry->has_external_query_at && !cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": somebody asked for block we didn't even tried to generate"; - } - cache_it = info.cache.erase(cache_it); - continue; - } - if (cache_entry->block_seqno == info.next_block_seqno && cached_prev != info.prev) { - cache_entry->cancel(td::Status::Error("invalid prev blocks")); - if (!cache_entry->has_external_query_at && cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": nobody asked for block, but we tried to generate it"; - } - if (cache_entry->has_external_query_at && !cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": somebody asked for block we didn't even tried to generate"; - } - cache_it = info.cache.erase(cache_it); - continue; - } - ++cache_it; - } - auto S = check_out_of_sync(); - if (S.is_error()) { - LOG(DEBUG) << "not generating block automatically: " << S; - return; - } - if (mc_config_status_.is_error()) { - LOG(DEBUG) << "not generating block automatically: unsupported mc config: " << mc_config_status_; - return; - } - generate_block(shard, cc_seqno, info.prev, {}, td::Timestamp::in(10.0), [](td::Result) {}); - } - return; - } - } - auto future_validator_group = get_future_validator_group(shard, cc_seqno); - if (future_validator_group.is_ok()) { - // future validator group, remember for later - future_validator_group.ok()->pending_blocks.push_back(std::move(prev)); - } -} - -td::Result CollatorNode::get_future_validator_group(ShardIdFull shard, - CatchainSeqno cc_seqno) { - auto it = validator_groups_.find(shard); - if (it == validator_groups_.end() && shard.pfx_len() != 0) { - it = validator_groups_.find(shard_parent(shard)); - } - if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) { - it = validator_groups_.find(shard_child(shard, true)); - } - if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) { - it = validator_groups_.find(shard_child(shard, false)); - } - if (it == validator_groups_.end()) { - return td::Status::Error("no such shard"); - } - if (cc_seqno < it->second.cc_seqno) { // past validator group - return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is outdated (current is" << it->second.cc_seqno - << ")"); - } - if (cc_seqno - it->second.cc_seqno > 1) { // future validator group, cc_seqno too big - return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is too big (currently known is" - << it->second.cc_seqno << ")"); - } - // future validator group - return &future_validator_groups_[{shard, cc_seqno}]; -} - -void CollatorNode::ValidatorGroupInfo::cleanup() { - prev.clear(); - next_block_seqno = 0; - for (auto& [_, cache_entry] : cache) { - cache_entry->cancel(td::Status::Error("validator group is outdated")); - } - cache.clear(); -} - -void CollatorNode::CacheEntry::cancel(td::Status reason) { - for (auto& promise : promises) { - promise.set_error(reason.clone()); - } - promises.clear(); - cancellation_token_source.cancel(); -} - -static td::BufferSlice serialize_error(td::Status error) { - return create_serialize_tl_object(error.code(), error.message().c_str()); -} - -static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator, CatchainSeqno& cc_seqno, - td::uint32& val_set_hash) { - CHECK(!block.id.is_masterchain()); - if (block.pubkey == creator) { - return block; - } - auto root = vm::std_boc_deserialize(block.data).move_as_ok(); - block::gen::Block::Record blk; - block::gen::BlockExtra::Record extra; - block::gen::BlockInfo::Record info; - CHECK(tlb::unpack_cell(root, blk)); - CHECK(tlb::unpack_cell(blk.extra, extra)); - CHECK(tlb::unpack_cell(blk.info, info)); - extra.created_by = creator.as_bits256(); - CHECK(tlb::pack_cell(blk.extra, extra)); - CHECK(tlb::pack_cell(root, blk)); - block.data = vm::std_boc_serialize(root, 31).move_as_ok(); - - block.id.root_hash = root->get_hash().bits(); - block.id.file_hash = block::compute_file_hash(block.data.as_slice()); - block.pubkey = creator; - - cc_seqno = info.gen_catchain_seqno; - val_set_hash = info.gen_validator_list_hash_short; - - for (auto& broadcast_ref : block.out_msg_queue_proof_broadcasts) { - auto block_state_proof = create_block_state_proof(root).move_as_ok(); - - auto& broadcast = broadcast_ref.write(); - broadcast.block_id = block.id; - broadcast.block_state_proofs = vm::std_boc_serialize(std::move(block_state_proof), 31).move_as_ok(); - } - return block; -} - -void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, - td::Promise promise) { - promise = [promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - if (R.error().code() == ErrorCode::timeout) { - promise.set_error(R.move_as_error()); - } else { - promise.set_result(serialize_error(R.move_as_error())); - } - } else { - promise.set_result(R.move_as_ok()); - } - }; - if (!opts_->check_collator_node_whitelist(src)) { - promise.set_error(td::Status::Error("not authorized")); - return; - } - if (!validator_adnl_ids_.contains(src)) { - promise.set_error(td::Status::Error("src is not a validator")); - return; - } - auto r_ping = fetch_tl_object(data, true); - if (r_ping.is_ok()) { - process_ping(src, *r_ping.ok_ref(), std::move(promise)); - return; - } - - TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); - ShardIdFull shard = create_shard_id(f->shard_); - CatchainSeqno cc_seqno = f->cc_seqno_; - std::vector prev_blocks; - for (const auto& b : f->prev_blocks_) { - prev_blocks.push_back(create_block_id(b)); - } - auto priority = BlockCandidatePriority{.round = static_cast(f->round_), - .first_block_round = static_cast(f->first_block_round_), - .priority = f->priority_}; - Ed25519_PublicKey creator(f->creator_); - td::Promise new_promise = [promise = std::move(promise), src, - shard](td::Result R) mutable { - if (R.is_error()) { - LOG(INFO) << "collate query from " << src << ", shard=" << shard.to_str() << ": error: " << R.error(); - promise.set_error(R.move_as_error()); - } else { - LOG(INFO) << "collate query from " << src << ", shard=" << shard.to_str() << ": success"; - promise.set_result(serialize_tl_object(serialize_candidate(R.move_as_ok(), true), true)); - } - }; - new_promise = [new_promise = std::move(new_promise), creator, local_id = local_id_, - manager = manager_](td::Result R) mutable { - TRY_RESULT_PROMISE(new_promise, block, std::move(R)); - - CollatorNodeResponseStats stats; - stats.self = local_id.pubkey_hash(); - stats.validator_id = PublicKey(pubkeys::Ed25519(creator)).compute_short_id(); - stats.original_block_id = block.id; - stats.collated_data_hash = block.collated_file_hash; - - CatchainSeqno cc_seqno; - td::uint32 val_set_hash; - block = change_creator(std::move(block), creator, cc_seqno, val_set_hash); - - stats.block_id = block.id; - stats.timestamp = td::Clocks::system(); - td::actor::send_closure(manager, &ValidatorManager::log_collator_node_response_stats, std::move(stats)); - - td::Promise P = - new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); }); - td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno, - val_set_hash, std::move(P)); - }; - if (!shard.is_valid_ext()) { - new_promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); - return; - } - if (prev_blocks.size() != 1 && prev_blocks.size() != 2) { - new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << prev_blocks.size())); - return; - } - LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno; - generate_block(shard, cc_seqno, std::move(prev_blocks), priority, td::Timestamp::in(10.0), std::move(new_promise)); -} - -void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - std::optional o_priority, td::Timestamp timeout, - td::Promise promise) { - bool is_external = !o_priority; - if (last_masterchain_state_.is_null()) { - promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); - return; - } - if (!can_collate_shard(shard)) { - promise.set_error(td::Status::Error(PSTRING() << "this node can't collate shard " << shard.to_str())); - return; - } - auto it = validator_groups_.find(shard); - if (it == validator_groups_.end() || it->second.cc_seqno != cc_seqno) { - TRY_RESULT_PROMISE(promise, future_validator_group, get_future_validator_group(shard, cc_seqno)); - future_validator_group->promises.push_back([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - if (timeout.is_in_past()) { - promise.set_error(td::Status::Error(ErrorCode::timeout)); - return; - } - td::actor::send_closure(SelfId, &CollatorNode::generate_block, shard, cc_seqno, std::move(prev_blocks), - std::move(o_priority), timeout, std::move(promise)); - }); - return; - } - ValidatorGroupInfo& validator_group_info = it->second; - BlockSeqno block_seqno = prev_blocks.at(0).seqno() + 1; - if (prev_blocks.size() == 2) { - block_seqno = std::max(block_seqno, prev_blocks.at(1).seqno() + 1); - } - if (validator_group_info.next_block_seqno > block_seqno) { - promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too small, expected " - << validator_group_info.next_block_seqno)); - return; - } - if (validator_group_info.next_block_seqno == block_seqno && validator_group_info.prev != prev_blocks) { - promise.set_error(td::Status::Error("invalid prev_blocks")); - return; - } - - static auto prefix_inner = [](auto& sb, auto& shard, auto cc_seqno, auto block_seqno, - const std::optional& o_priority) { - sb << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; - if (o_priority) { - sb << " external{"; - sb << "round_offset=" << o_priority->round - o_priority->first_block_round - << ",priority=" << o_priority->priority; - sb << ",first_block_round=" << o_priority->first_block_round; - sb << "}"; - } else { - sb << " internal"; - } - }; - auto prefix = [&](auto& sb) { prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); }; - - auto cache_entry = validator_group_info.cache[prev_blocks]; - if (cache_entry == nullptr) { - cache_entry = validator_group_info.cache[prev_blocks] = std::make_shared(); - } - if (is_external && !cache_entry->has_external_query_at) { - cache_entry->has_external_query_at = td::Timestamp::now(); - if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { - FLOG(INFO) { - prefix(sb); - sb << ": got external query " - << cache_entry->has_external_query_at.at() - cache_entry->has_internal_query_at.at() - << "s after internal query [WON]"; - }; - } - } - if (!is_external && !cache_entry->has_internal_query_at) { - cache_entry->has_internal_query_at = td::Timestamp::now(); - if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { - FLOG(INFO) { - prefix(sb); - sb << ": got internal query " - << cache_entry->has_internal_query_at.at() - cache_entry->has_external_query_at.at() - << "s after external query [LOST]"; - }; - } - } - if (cache_entry->result) { - auto has_result_ago = td::Timestamp::now().at() - cache_entry->has_result_at.at(); - FLOG(INFO) { - prefix(sb); - sb << ": using cached result " << " generated " << has_result_ago << "s ago"; - sb << (is_external ? " for external query [WON]" : " for internal query "); - }; - - promise.set_result(cache_entry->result.value().clone()); - return; - } - cache_entry->promises.push_back(std::move(promise)); - - if (cache_entry->started) { - FLOG(INFO) { - prefix(sb); - sb << ": collation in progress, waiting"; - }; - return; - } - FLOG(INFO) { - prefix(sb); - sb << ": starting collation"; - }; - cache_entry->started = true; - cache_entry->block_seqno = block_seqno; - run_collate_query( - shard, last_masterchain_state_->get_block_id(), std::move(prev_blocks), Ed25519_PublicKey{td::Bits256::zero()}, - last_masterchain_state_->get_validator_set(shard), opts_->get_collator_options(), manager_, timeout, - [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { - FLOG(INFO) { - prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); - sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); - }; - td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); - }, - local_id_, cache_entry->cancellation_token_source.get_cancellation_token(), - CollateMode::skip_store_candidate | CollateMode::from_collator_node); -} - -void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { - if (R.is_error()) { - cache_entry->started = false; - for (auto& p : cache_entry->promises) { - p.set_error(R.error().clone()); - } - } else { - cache_entry->result = R.move_as_ok(); - cache_entry->has_result_at = td::Timestamp::now(); - for (auto& p : cache_entry->promises) { - p.set_result(cache_entry->result.value().clone()); - } - } - cache_entry->promises.clear(); -} - -td::Status CollatorNode::check_out_of_sync() { - if (last_masterchain_state_.is_null() || !shard_client_handle_) { - return td::Status::Error("not inited"); - } - auto now = (UnixTime)td::Clocks::system(); - if (last_masterchain_state_->get_unix_time() < now - 60 || shard_client_handle_->unix_time() < now - 60) { - return td::Status::Error(PSTRING() << "out of sync: mc " << now - last_masterchain_state_->get_unix_time() - << "s ago, shardclient " << now - shard_client_handle_->unix_time() << "s ago"); - } - return td::Status::OK(); -} - -td::Status CollatorNode::check_mc_config() { - if (last_masterchain_state_.is_null()) { - return td::Status::Error("not inited"); - } - TRY_RESULT_PREFIX( - config, - block::ConfigInfo::extract_config(last_masterchain_state_->root_cell(), block::ConfigInfo::needCapabilities), - "cannot unpack masterchain config"); - if (config->get_global_version() > Collator::supported_version()) { - return td::Status::Error(PSTRING() << "unsupported global version " << config->get_global_version() - << " (supported: " << Collator::supported_version() << ")"); - } - if (config->get_capabilities() & ~Collator::supported_capabilities()) { - return td::Status::Error(PSTRING() << "unsupported capabilities " << config->get_capabilities() - << " (supported: " << Collator::supported_capabilities() << ")"); - } - td::Status S = td::Status::OK(); - config->foreach_config_param([&](int idx, td::Ref param) { - if (idx < 0) { - return true; - } - if (!block::gen::ConfigParam{idx}.validate_ref(1024, std::move(param))) { - S = td::Status::Error(PSTRING() << "unknown ConfigParam " << idx); - return false; - } - return true; - }); - return S; -} - -void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, - td::Promise promise) { - LOG(DEBUG) << "got ping from " << src; - TRY_STATUS_PROMISE(promise, check_out_of_sync()); - TRY_STATUS_PROMISE_PREFIX(promise, mc_config_status_.clone(), "unsupported mc config: "); - promise.set_result(create_serialize_tl_object(0)); -} - -bool CollatorNode::can_collate_shard(ShardIdFull shard) const { - return std::any_of(collating_shards_.begin(), collating_shards_.end(), - [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); -} - -tl_object_ptr CollatorNode::serialize_candidate(const BlockCandidate& block, - bool compress) { - if (!compress) { - return create_tl_object( - PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), block.data.clone(), - block.collated_data.clone()); - } - size_t decompressed_size; - td::BufferSlice compressed = - validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok(); - return create_tl_object( - 0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), - (int)decompressed_size, std::move(compressed)); -} - -td::Result CollatorNode::deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size, int proto_version) { - td::Result res; - ton_api::downcast_call(*f, td::overloaded( - [&](ton_api::collatorNode_candidate& c) { - res = [&]() -> td::Result { - auto hash = td::sha256_bits256(c.collated_data_); - auto key = ton::PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), - std::move(c.collated_data_)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidate& c) { - res = [&]() -> td::Result { - if (c.decompressed_size_ <= 0) { - return td::Status::Error("invalid decompressed size"); - } - if (c.decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, validatorsession::decompress_candidate_data( - c.data_, c.decompressed_size_, proto_version)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = ton::PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, - std::move(p.first), std::move(p.second)}; - }(); - })); - return res; -} - -} // namespace ton::validator diff --git a/validator/collator-node/collator-node-session.cpp b/validator/collator-node/collator-node-session.cpp new file mode 100644 index 000000000..8e131f3c0 --- /dev/null +++ b/validator/collator-node/collator-node-session.cpp @@ -0,0 +1,314 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "collator-node-session.hpp" + +#include "collator-node.hpp" +#include "fabric.h" +#include "utils.hpp" + +namespace ton::validator { + +static BlockSeqno get_next_block_seqno(const std::vector& prev) { + if (prev.size() == 1) { + return prev[0].seqno() + 1; + } + CHECK(prev.size() == 2); + return std::max(prev[0].seqno(), prev[1].seqno()) + 1; +} + +CollatorNodeSession::CollatorNodeSession(ShardIdFull shard, std::vector prev, + td::Ref validator_set, BlockIdExt min_masterchain_block_id, + bool can_generate, Ref state, adnl::AdnlNodeIdShort local_id, + td::Ref opts, + td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : shard_(shard) + , prev_(std::move(prev)) + , validator_set_(validator_set) + , min_masterchain_block_id_(min_masterchain_block_id) + , can_generate_(can_generate) + , local_id_(local_id) + , opts_(opts) + , manager_(manager) + , adnl_(adnl) + , rldp_(rldp) + , next_block_seqno_(get_next_block_seqno(prev_)) { + update_masterchain_config(state); +} + +void CollatorNodeSession::start_up() { + LOG(INFO) << "Starting collator node session, shard " << shard_.to_str() << ", cc_seqno " + << validator_set_->get_catchain_seqno() << ", next block seqno " << next_block_seqno_; + + if (can_generate_) { + generate_block(prev_, {}, {}, td::Timestamp::in(10.0), [](td::Result) {}); + } +} + +void CollatorNodeSession::tear_down() { + LOG(INFO) << "Finishing collator node session, shard " << shard_.to_str() << ", cc_seqno " + << validator_set_->get_catchain_seqno(); + for (auto& [_, entry] : cache_) { + entry->cancel(td::Status::Error("validator session finished")); + } +} + +void CollatorNodeSession::new_shard_block_accepted(BlockIdExt block_id, bool can_generate) { + CHECK(block_id.shard_full() == shard_); + can_generate_ = can_generate; + if (next_block_seqno_ > block_id.seqno()) { + return; + } + LOG(DEBUG) << "New shard block " << block_id.to_str(); + next_block_seqno_ = block_id.seqno() + 1; + prev_ = {block_id}; + + while (!cache_.empty()) { + auto& [cache_prev, entry] = *cache_.begin(); + if (entry->block_seqno < next_block_seqno_) { + entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << entry->block_seqno << " is too old, expected " + << next_block_seqno_)); + } else if (entry->block_seqno == next_block_seqno_ && prev_ != cache_prev) { + entry->cancel(td::Status::Error(PSTRING() << "invalid prev blocks for seqno " << entry->block_seqno)); + } else { + break; + } + if (!entry->has_external_query_at && entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard_.to_str() << ", cc_seqno=" << validator_set_->get_catchain_seqno() + << ", next_block_seqno=" << entry->block_seqno + << ": nobody asked for block, but we tried to generate it"; + } + if (entry->has_external_query_at && !entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard_.to_str() << ", cc_seqno=" << validator_set_->get_catchain_seqno() + << ", next_block_seqno=" << entry->block_seqno + << ": somebody asked for block we didn't even try to generate"; + } + cache_.erase(cache_.begin()); + } + + if (can_generate_) { + generate_block(prev_, {}, {}, td::Timestamp::in(10.0), [](td::Result) {}); + } +} + +void CollatorNodeSession::update_masterchain_config(td::Ref state) { + ValidatorSessionConfig config = state->get_consensus_config(); + proto_version_ = config.proto_version; + max_candidate_size_ = config.max_block_size + config.max_collated_data_size + 1024; +} + +void CollatorNodeSession::generate_block(std::vector prev_blocks, + td::optional o_priority, + td::Ref o_optimistic_prev_block, td::Timestamp timeout, + td::Promise promise) { + bool is_external = !o_priority; + bool is_optimistic = o_optimistic_prev_block.not_null(); + BlockSeqno block_seqno = get_next_block_seqno(prev_blocks); + if (next_block_seqno_ > block_seqno) { + promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too old, expected " + << next_block_seqno_)); + return; + } + if (next_block_seqno_ == block_seqno && prev_ != prev_blocks) { + promise.set_error(td::Status::Error("invalid prev_blocks")); + return; + } + if (next_block_seqno_ + 10 < block_seqno) { + promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too new, current is " + << next_block_seqno_)); + return; + } + + static auto prefix_inner = [](td::StringBuilder& sb, const ShardIdFull& shard, CatchainSeqno cc_seqno, + BlockSeqno block_seqno, const td::optional& o_priority, + bool is_optimistic) { + sb << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; + if (o_priority) { + sb << " external{"; + sb << "round_offset=" << o_priority.value().round - o_priority.value().first_block_round + << ",priority=" << o_priority.value().priority; + sb << ",first_block_round=" << o_priority.value().first_block_round; + sb << "}"; + } else { + sb << " internal"; + } + if (is_optimistic) { + sb << " opt"; + } + }; + auto prefix = [&](td::StringBuilder& sb) { + prefix_inner(sb, shard_, validator_set_->get_catchain_seqno(), block_seqno, o_priority, is_optimistic); + }; + + auto cache_entry = cache_[prev_blocks]; + if (cache_entry == nullptr) { + cache_entry = cache_[prev_blocks] = std::make_shared(); + } + if (is_external && !cache_entry->has_external_query_at) { + cache_entry->has_external_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got external query " << cache_entry->has_external_query_at - cache_entry->has_internal_query_at + << "s after internal query [WON]"; + }; + } + } + if (!is_external && !cache_entry->has_internal_query_at) { + cache_entry->has_internal_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got internal query " << cache_entry->has_internal_query_at - cache_entry->has_external_query_at + << "s after external query [LOST]"; + }; + } + } + if (cache_entry->result) { + auto has_result_ago = td::Timestamp::now() - cache_entry->has_result_at; + FLOG(INFO) { + prefix(sb); + sb << ": using cached result " << " generated " << has_result_ago << "s ago"; + sb << (is_external ? " for external query [WON]" : " for internal query "); + }; + promise.set_result(cache_entry->result.value().clone()); + return; + } + cache_entry->promises.push_back(std::move(promise)); + + if (cache_entry->started) { + FLOG(INFO) { + prefix(sb); + sb << ": collation in progress, waiting"; + }; + return; + } + FLOG(INFO) { + prefix(sb); + sb << ": starting collation"; + }; + cache_entry->started = true; + cache_entry->block_seqno = block_seqno; + run_collate_query(CollateParams{.shard = shard_, + .min_masterchain_block_id = min_masterchain_block_id_, + .prev = std::move(prev_blocks), + .validator_set = validator_set_, + .collator_opts = opts_->get_collator_options(), + .collator_node_id = local_id_, + .skip_store_candidate = true, + .optimistic_prev_block = o_optimistic_prev_block}, + manager_, timeout, cache_entry->cancellation_token_source.get_cancellation_token(), + [=, shard = shard_, cc_seqno = validator_set_->get_catchain_seqno(), SelfId = actor_id(this), + timer = td::Timer{}](td::Result R) mutable { + FLOG(INFO) { + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority, is_optimistic); + sb << ": " << (R.is_ok() ? "OK" : R.error().to_string()) << " time=" << timer.elapsed(); + }; + td::actor::send_closure(SelfId, &CollatorNodeSession::process_result, cache_entry, std::move(R)); + }); +} + +void CollatorNodeSession::process_result(std::shared_ptr cache_entry, td::Result R) { + if (R.is_error()) { + cache_entry->started = false; + for (auto& p : cache_entry->promises) { + p.set_error(R.error().clone()); + } + } else { + cache_entry->result = R.move_as_ok(); + cache_entry->has_result_at = td::Timestamp::now(); + for (auto& p : cache_entry->promises) { + p.set_result(cache_entry->result.value().clone()); + } + } + cache_entry->promises.clear(); +} + +void CollatorNodeSession::process_request(adnl::AdnlNodeIdShort src, std::vector prev_blocks, + BlockCandidatePriority priority, bool is_optimistic, td::Timestamp timeout, + td::Promise promise) { + if (is_optimistic) { + if (prev_blocks.size() != 1) { + promise.set_error(td::Status::Error("optimistic collation, expected 1 prev block")); + return; + } + auto it = cache_.find(prev_blocks); + if (it == cache_.end() || it->second->started) { + BlockIdExt prev_block = prev_blocks[0]; + td::actor::send_closure( + manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, prev_block, + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollatorNodeSession::process_request_optimistic_cont, src, prev_block, + priority, timeout, std::move(promise), std::move(R)); + }); + return; + } + } + generate_block(std::move(prev_blocks), priority, {}, timeout, std::move(promise)); +} + +void CollatorNodeSession::process_request_optimistic_cont(adnl::AdnlNodeIdShort src, BlockIdExt prev_block_id, + BlockCandidatePriority priority, td::Timestamp timeout, + td::Promise promise, + td::Result prev_block_data) { + if (prev_block_data.is_ok()) { + TRY_RESULT_PROMISE_PREFIX(promise, prev_block, create_block(prev_block_id, prev_block_data.move_as_ok()), + "invalid prev block data in db: "); + LOG(INFO) << "got prev block from db for optimistic collation: " << prev_block_id.to_str(); + generate_block({prev_block_id}, priority, prev_block, timeout, std::move(promise)); + return; + } + td::actor::send_closure( + rldp_, &rldp2::Rldp::send_query_ex, local_id_, src, "getprevblock", + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollatorNodeSession::process_request_optimistic_cont2, prev_block_id, priority, + timeout, std::move(promise), std::move(R)); + }, + timeout, + create_serialize_tl_object(0, create_tl_block_id(prev_block_id)), + max_candidate_size_); +} + +void CollatorNodeSession::process_request_optimistic_cont2(BlockIdExt prev_block_id, BlockCandidatePriority priority, + td::Timestamp timeout, td::Promise promise, + td::Result R) { + TRY_RESULT_PROMISE_PREFIX(promise, response, std::move(R), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, f, fetch_tl_object(response, true), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, candidate, + deserialize_candidate(std::move(f), max_candidate_size_, proto_version_), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, prev_block, create_block(prev_block_id, std::move(candidate.data)), + "invalid prev block data from validator: "); + LOG(INFO) << "got prev block from validator for optimistic collation: " << prev_block_id.to_str(); + generate_block({prev_block_id}, priority, prev_block, timeout, std::move(promise)); +} + +void CollatorNodeSession::CacheEntry::cancel(td::Status reason) { + for (auto& promise : promises) { + promise.set_error(reason.clone()); + } + promises.clear(); + cancellation_token_source.cancel(); +} + +} // namespace ton::validator diff --git a/validator/collator-node/collator-node-session.hpp b/validator/collator-node/collator-node-session.hpp new file mode 100644 index 000000000..0f9f03d30 --- /dev/null +++ b/validator/collator-node/collator-node-session.hpp @@ -0,0 +1,95 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "interfaces/validator-manager.h" +#include "rldp/rldp.h" +#include "rldp2/rldp.h" +#include +#include + +namespace ton::validator { + +class ValidatorManager; + +class CollatorNodeSession : public td::actor::Actor { + public: + CollatorNodeSession(ShardIdFull shard, std::vector prev, td::Ref validator_set, + BlockIdExt min_masterchain_block_id, bool can_generate, td::Ref state, + adnl::AdnlNodeIdShort local_id, td::Ref opts, + td::actor::ActorId manager, td::actor::ActorId adnl, + td::actor::ActorId rldp); + + void start_up() override; + void tear_down() override; + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + + void new_shard_block_accepted(BlockIdExt block_id, bool can_generate); + + void process_request(adnl::AdnlNodeIdShort src, std::vector prev_blocks, BlockCandidatePriority priority, + bool is_optimistic, td::Timestamp timeout, td::Promise promise); + void update_masterchain_config(td::Ref state); + + private: + ShardIdFull shard_; + std::vector prev_; + td::Ref validator_set_; + BlockIdExt min_masterchain_block_id_; + bool can_generate_; + adnl::AdnlNodeIdShort local_id_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + struct CacheEntry { + bool started = false; + td::Timestamp has_internal_query_at; + td::Timestamp has_external_query_at; + td::Timestamp has_result_at; + BlockSeqno block_seqno = 0; + td::optional result; + td::CancellationTokenSource cancellation_token_source; + std::vector> promises; + + void cancel(td::Status reason); + }; + + BlockSeqno next_block_seqno_; + std::map, std::shared_ptr> cache_; + + td::uint32 proto_version_ = 0; + td::uint32 max_candidate_size_ = 0; + + void generate_block(std::vector prev_blocks, td::optional o_priority, + td::Ref o_optimistic_prev_block, td::Timestamp timeout, + td::Promise promise); + void process_result(std::shared_ptr cache_entry, td::Result R); + + void process_request_optimistic_cont(adnl::AdnlNodeIdShort src, BlockIdExt prev_block_id, + BlockCandidatePriority priority, td::Timestamp timeout, + td::Promise promise, + td::Result prev_block_data); + void process_request_optimistic_cont2(BlockIdExt prev_block_id, BlockCandidatePriority priority, + td::Timestamp timeout, td::Promise promise, + td::Result R); +}; + +} // namespace ton::validator diff --git a/validator/collator-node/collator-node.cpp b/validator/collator-node/collator-node.cpp new file mode 100644 index 000000000..9b03d9dc0 --- /dev/null +++ b/validator/collator-node/collator-node.cpp @@ -0,0 +1,511 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "collator-node.hpp" +#include "ton/ton-tl.hpp" +#include "fabric.h" +#include "block-auto.h" +#include "block-db.h" +#include "td/utils/lz4.h" +#include "checksum.h" +#include "impl/collator-impl.h" +#include "impl/shard.hpp" +#include "utils.hpp" + +namespace ton::validator { + +CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::Ref opts, + td::actor::ActorId manager, td::actor::ActorId adnl, + td::actor::ActorId rldp) + : local_id_(local_id) + , opts_(std::move(opts)) + , manager_(std::move(manager)) + , adnl_(std::move(adnl)) + , rldp_(std::move(rldp)) { +} + +void CollatorNode::start_up() { + class Cb : public adnl::Adnl::Callback { + public: + explicit Cb(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &CollatorNode::receive_query, src, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlockOptimistic::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); +} + +void CollatorNode::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlockOptimistic::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID)); +} + +void CollatorNode::add_shard(ShardIdFull shard) { + CHECK(shard.is_valid_ext() && !shard.is_masterchain()); + if (std::find(collating_shards_.begin(), collating_shards_.end(), shard) != collating_shards_.end()) { + return; + } + LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); + collating_shards_.push_back(shard); + if (last_masterchain_state_.is_null()) { + return; + } + for (auto& [group_shard, validator_group] : validator_groups_) { + if (validator_group.actor.empty() && shard_intersects(shard, group_shard)) { + validator_group.actor = td::actor::create_actor( + PSTRING() << "collatornode" << shard.to_str(), shard, validator_group.prev, + last_masterchain_state_->get_validator_set(group_shard), last_masterchain_state_->get_block_id(), + can_generate(), last_masterchain_state_, local_id_, opts_, manager_, adnl_, rldp_); + } + } +} + +void CollatorNode::del_shard(ShardIdFull shard) { + auto it = std::find(collating_shards_.begin(), collating_shards_.end(), shard); + if (it != collating_shards_.end()) { + collating_shards_.erase(it); + } + for (auto& [group_shard, validator_group] : validator_groups_) { + if (!validator_group.actor.empty() && shard_intersects(shard, group_shard) && !can_collate_shard(group_shard)) { + validator_group.actor = {}; + } + } +} + +void CollatorNode::update_options(td::Ref opts) { + for (auto& [_, shard] : validator_groups_) { + if (!shard.actor.empty()) { + td::actor::send_closure(shard.actor, &CollatorNodeSession::update_options, opts); + } + } + opts_ = std::move(opts); +} + +void CollatorNode::new_masterchain_block_notification(td::Ref state) { + last_masterchain_state_ = state; + + if (state->last_key_block_id().seqno() != last_key_block_seqno_) { + last_key_block_seqno_ = state->last_key_block_id().seqno(); + mc_config_status_ = check_mc_config(); + if (mc_config_status_.is_error()) { + LOG(ERROR) << "Cannot validate masterchain config (possibly outdated software): " << mc_config_status_; + } + + validator_adnl_ids_.clear(); + for (int next : {-1, 0, 1}) { + td::Ref vals = state->get_total_validator_set(next); + if (vals.not_null()) { + for (const ValidatorDescr& descr : vals->export_vector()) { + if (descr.addr.is_zero()) { + validator_adnl_ids_.insert( + adnl::AdnlNodeIdShort(PublicKey(pubkeys::Ed25519{descr.key.as_bits256()}).compute_short_id())); + } else { + validator_adnl_ids_.insert(adnl::AdnlNodeIdShort(descr.addr)); + } + } + } + } + for (auto& [_, group] : validator_groups_) { + if (!group.actor.empty()) { + td::actor::send_closure(group.actor, &CollatorNodeSession::update_masterchain_config, state); + } + } + } + + std::map> new_shards; + for (auto& v : state->get_shards()) { + auto shard = v->shard(); + if (v->before_split()) { + CHECK(!v->before_merge()); + new_shards.emplace(shard_child(shard, true), std::vector{v->top_block_id()}); + new_shards.emplace(shard_child(shard, false), std::vector{v->top_block_id()}); + } else if (v->before_merge()) { + ShardIdFull p_shard = shard_parent(shard); + auto it = new_shards.find(p_shard); + if (it == new_shards.end()) { + new_shards.emplace(p_shard, std::vector(2)); + } + bool left = shard_child(p_shard.shard, true) == shard.shard; + new_shards[p_shard][left ? 0 : 1] = v->top_block_id(); + } else { + new_shards.emplace(shard, std::vector{v->top_block_id()}); + } + } + for (auto it = validator_groups_.begin(); it != validator_groups_.end();) { + if (new_shards.contains(it->first)) { + ++it; + } else { + it = validator_groups_.erase(it); + } + } + for (auto& [shard, prev] : new_shards) { + auto validator_set = state->get_validator_set(shard); + CatchainSeqno cc_seqno = validator_set->get_catchain_seqno(); + auto [it, created] = validator_groups_.emplace(shard, ValidatorGroupInfo{}); + it->second.prev = std::move(prev); + if (created || it->second.cc_seqno != cc_seqno) { + it->second.cc_seqno = cc_seqno; + if (can_collate_shard(shard)) { + it->second.actor = td::actor::create_actor( + PSTRING() << "collatornode" << shard.to_str(), shard, it->second.prev, validator_set, + last_masterchain_state_->get_block_id(), can_generate(), last_masterchain_state_, local_id_, opts_, + manager_, adnl_, rldp_); + } + } else if (!it->second.actor.empty() && prev.size() == 1) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, prev[0], + can_generate()); + } + auto it2 = future_validator_groups_.find({shard, cc_seqno}); + if (it2 != future_validator_groups_.end()) { + FutureValidatorGroup& future_group = it2->second; + if (!it->second.actor.empty()) { + for (const BlockIdExt& block_id : future_group.pending_blocks) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, block_id, + can_generate()); + } + } + for (auto& promise : future_group.promises) { + promise.set_value(td::Unit()); + } + future_validator_groups_.erase(it2); + } + } + + for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end();) { + if (get_future_validator_group(it->first.first, it->first.second).is_ok()) { + ++it; + } else { + auto& future_group = it->second; + for (auto& promise : future_group.promises) { + promise.set_error(td::Status::Error("validator group is outdated")); + } + it = future_validator_groups_.erase(it); + } + } +} + +void CollatorNode::update_shard_client_handle(BlockHandle shard_client_handle) { + shard_client_handle_ = shard_client_handle; +} + +void CollatorNode::new_shard_block_accepted(BlockIdExt block_id, CatchainSeqno cc_seqno) { + if (!can_collate_shard(block_id.shard_full())) { + return; + } + auto it = validator_groups_.find(block_id.shard_full()); + if (it == validator_groups_.end() || it->second.cc_seqno != cc_seqno) { + auto future_group = get_future_validator_group(block_id.shard_full(), cc_seqno); + if (future_group.is_error()) { + LOG(DEBUG) << "Dropping new shard block " << block_id.to_str() << " cc_seqno=" << cc_seqno << " : " + << future_group.error(); + } else { + LOG(DEBUG) << "New shard block in future validator group " << block_id.to_str() << " cc_seqno=" << cc_seqno; + future_group.ok()->pending_blocks.push_back(block_id); + } + return; + } + if (!it->second.actor.empty()) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, block_id, can_generate()); + } +} + +td::Result CollatorNode::get_future_validator_group(ShardIdFull shard, + CatchainSeqno cc_seqno) { + auto it = validator_groups_.find(shard); + if (it == validator_groups_.end() && shard.pfx_len() != 0) { + it = validator_groups_.find(shard_parent(shard)); + } + if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) { + it = validator_groups_.find(shard_child(shard, true)); + } + if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) { + it = validator_groups_.find(shard_child(shard, false)); + } + if (it == validator_groups_.end()) { + return td::Status::Error("no such shard"); + } + if (cc_seqno < it->second.cc_seqno) { // past validator group + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() + << " is outdated (current is " << it->second.cc_seqno << ")"); + } + if (cc_seqno - it->second.cc_seqno > 1) { // future validator group, cc_seqno too big + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() + << " is too big (currently known is " << it->second.cc_seqno << ")"); + } + // future validator group + return &future_validator_groups_[{shard, cc_seqno}]; +} + +static td::BufferSlice serialize_error(td::Status error) { + return create_serialize_tl_object(error.code(), error.message().c_str()); +} + +static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator, CatchainSeqno& cc_seqno, + td::uint32& val_set_hash) { + CHECK(!block.id.is_masterchain()); + if (block.pubkey == creator) { + return block; + } + auto root = vm::std_boc_deserialize(block.data).move_as_ok(); + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + block::gen::BlockInfo::Record info; + CHECK(tlb::unpack_cell(root, blk)); + CHECK(tlb::unpack_cell(blk.extra, extra)); + CHECK(tlb::unpack_cell(blk.info, info)); + extra.created_by = creator.as_bits256(); + CHECK(tlb::pack_cell(blk.extra, extra)); + CHECK(tlb::pack_cell(root, blk)); + block.data = vm::std_boc_serialize(root, 31).move_as_ok(); + + block.id.root_hash = root->get_hash().bits(); + block.id.file_hash = block::compute_file_hash(block.data.as_slice()); + block.pubkey = creator; + + cc_seqno = info.gen_catchain_seqno; + val_set_hash = info.gen_validator_list_hash_short; + + for (auto& broadcast_ref : block.out_msg_queue_proof_broadcasts) { + auto block_state_proof = create_block_state_proof(root).move_as_ok(); + + auto& broadcast = broadcast_ref.write(); + broadcast.block_id = block.id; + broadcast.block_state_proofs = vm::std_boc_serialize(std::move(block_state_proof), 31).move_as_ok(); + } + return block; +} + +void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, + td::Promise promise) { + promise = [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + if (R.error().code() == ErrorCode::timeout) { + promise.set_error(R.move_as_error()); + } else { + promise.set_result(serialize_error(R.move_as_error())); + } + } else { + promise.set_result(R.move_as_ok()); + } + }; + if (!opts_->check_collator_node_whitelist(src)) { + promise.set_error(td::Status::Error("not authorized")); + return; + } + if (!validator_adnl_ids_.contains(src)) { + promise.set_error(td::Status::Error("src is not a validator")); + return; + } + auto r_ping = fetch_tl_object(data, true); + if (r_ping.is_ok()) { + process_ping(src, *r_ping.ok_ref(), std::move(promise)); + return; + } + + bool is_optimistic = false; + ShardIdFull shard; + CatchainSeqno cc_seqno; + std::vector prev_blocks; + BlockCandidatePriority priority; + Ed25519_PublicKey creator; + if (auto R = fetch_tl_object(data, true); R.is_ok()) { + auto f = R.move_as_ok(); + shard = create_shard_id(f->shard_); + cc_seqno = f->cc_seqno_; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + priority = BlockCandidatePriority{.round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_}; + creator = Ed25519_PublicKey(f->creator_); + } else if (auto R = fetch_tl_object(data, true); R.is_ok()) { + is_optimistic = true; + auto f = R.move_as_ok(); + shard = create_shard_id(f->shard_); + cc_seqno = f->cc_seqno_; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + priority = BlockCandidatePriority{.round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_}; + creator = Ed25519_PublicKey(f->creator_); + } else { + promise.set_error(td::Status::Error("cannot parse request")); + return; + } + td::Promise new_promise = [promise = std::move(promise), src, + shard](td::Result R) mutable { + if (R.is_error()) { + LOG(INFO) << "collate query from " << src << ", shard=" << shard.to_str() << ": error: " << R.error(); + promise.set_error(R.move_as_error()); + } else { + LOG(INFO) << "collate query from " << src << ", shard=" << shard.to_str() << ": success"; + promise.set_result(serialize_tl_object(serialize_candidate(R.move_as_ok(), true), true)); + } + }; + new_promise = [new_promise = std::move(new_promise), creator, local_id = local_id_, + manager = manager_](td::Result R) mutable { + TRY_RESULT_PROMISE(new_promise, block, std::move(R)); + + CollatorNodeResponseStats stats; + stats.self = local_id.pubkey_hash(); + stats.validator_id = PublicKey(pubkeys::Ed25519(creator)).compute_short_id(); + stats.original_block_id = block.id; + stats.collated_data_hash = block.collated_file_hash; + + CatchainSeqno cc_seqno; + td::uint32 val_set_hash; + block = change_creator(std::move(block), creator, cc_seqno, val_set_hash); + + stats.block_id = block.id; + stats.timestamp = td::Clocks::system(); + td::actor::send_closure(manager, &ValidatorManager::log_collator_node_response_stats, std::move(stats)); + + td::Promise P = + new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); }); + td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno, + val_set_hash, std::move(P)); + }; + if (!shard.is_valid_ext()) { + new_promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); + return; + } + if (prev_blocks.size() != 1 && prev_blocks.size() != 2) { + new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << prev_blocks.size())); + return; + } + LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << (is_optimistic ? ", optimistic" : ""); + process_generate_block_query(src, shard, cc_seqno, std::move(prev_blocks), priority, is_optimistic, + td::Timestamp::in(10.0), std::move(new_promise)); +} + +void CollatorNode::process_generate_block_query(adnl::AdnlNodeIdShort src, ShardIdFull shard, CatchainSeqno cc_seqno, + std::vector prev_blocks, BlockCandidatePriority priority, + bool is_optimistic, td::Timestamp timeout, + td::Promise promise) { + if (last_masterchain_state_.is_null()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); + return; + } + if (timeout.is_in_past()) { + promise.set_error(td::Status::Error(ErrorCode::timeout)); + return; + } + auto it = validator_groups_.find(shard); + if (it == validator_groups_.end() || it->second.cc_seqno != cc_seqno) { + TRY_RESULT_PROMISE(promise, future_validator_group, get_future_validator_group(shard, cc_seqno)); + future_validator_group->promises.push_back([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), + promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &CollatorNode::process_generate_block_query, src, shard, cc_seqno, + std::move(prev_blocks), std::move(priority), is_optimistic, timeout, std::move(promise)); + }); + return; + } + ValidatorGroupInfo& validator_group_info = it->second; + if (validator_group_info.actor.empty()) { + promise.set_error(td::Status::Error(PSTRING() << "cannot collate shard " << shard.to_str())); + return; + } + td::actor::send_closure(validator_group_info.actor, &CollatorNodeSession::process_request, src, + std::move(prev_blocks), priority, is_optimistic, timeout, std::move(promise)); +} + +td::Status CollatorNode::check_out_of_sync() { + if (last_masterchain_state_.is_null() || !shard_client_handle_) { + return td::Status::Error("not inited"); + } + auto now = (UnixTime)td::Clocks::system(); + if (last_masterchain_state_->get_unix_time() < now - 60 || shard_client_handle_->unix_time() < now - 60) { + return td::Status::Error(PSTRING() << "out of sync: mc " << now - last_masterchain_state_->get_unix_time() + << "s ago, shardclient " << now - shard_client_handle_->unix_time() << "s ago"); + } + return td::Status::OK(); +} + +td::Status CollatorNode::check_mc_config() { + if (last_masterchain_state_.is_null()) { + return td::Status::Error("not inited"); + } + TRY_RESULT_PREFIX( + config, + block::ConfigInfo::extract_config(last_masterchain_state_->root_cell(), last_masterchain_state_->get_block_id(), + block::ConfigInfo::needCapabilities), + "cannot unpack masterchain config"); + if (config->get_global_version() > Collator::supported_version()) { + return td::Status::Error(PSTRING() << "unsupported global version " << config->get_global_version() + << " (supported: " << Collator::supported_version() << ")"); + } + if (config->get_capabilities() & ~Collator::supported_capabilities()) { + return td::Status::Error(PSTRING() << "unsupported capabilities " << config->get_capabilities() + << " (supported: " << Collator::supported_capabilities() << ")"); + } + td::Status S = td::Status::OK(); + config->foreach_config_param([&](int idx, td::Ref param) { + if (idx < 0) { + return true; + } + if (!block::gen::ConfigParam{idx}.validate_ref(1024, std::move(param))) { + S = td::Status::Error(PSTRING() << "unknown ConfigParam " << idx); + return false; + } + return true; + }); + return S; +} + +void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, + td::Promise promise) { + LOG(DEBUG) << "got ping from " << src; + TRY_STATUS_PROMISE(promise, check_out_of_sync()); + TRY_STATUS_PROMISE_PREFIX(promise, mc_config_status_.clone(), "unsupported mc config: "); + auto pong = create_tl_object(); + if (ping.flags_ & ton_api::collatorNode_pong::VERSION_MASK) { + pong->flags_ |= ton_api::collatorNode_pong::VERSION_MASK; + pong->version_ = COLLATOR_NODE_VERSION; + } + promise.set_result(serialize_tl_object(pong, true)); +} + +bool CollatorNode::can_collate_shard(ShardIdFull shard) const { + return std::any_of(collating_shards_.begin(), collating_shards_.end(), + [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); +} + +} // namespace ton::validator diff --git a/validator/collator-node.hpp b/validator/collator-node/collator-node.hpp similarity index 66% rename from validator/collator-node.hpp rename to validator/collator-node/collator-node.hpp index a4df03681..cb9cec47c 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node/collator-node.hpp @@ -16,6 +16,7 @@ */ #pragma once +#include "collator-node-session.hpp" #include "interfaces/validator-manager.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" @@ -36,16 +37,17 @@ class CollatorNode : public td::actor::Actor { void add_shard(ShardIdFull shard); void del_shard(ShardIdFull shard); + void update_options(td::Ref opts); + void new_masterchain_block_notification(td::Ref state); void update_shard_client_handle(BlockHandle shard_client_handle); - void update_validator_group_info(ShardIdFull shard, std::vector prev, CatchainSeqno cc_seqno); - - void update_options(td::Ref opts) { - opts_ = std::move(opts); - } + void new_shard_block_accepted(BlockIdExt block_id, CatchainSeqno cc_seqno); private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + void process_generate_block_query(adnl::AdnlNodeIdShort src, ShardIdFull shard, CatchainSeqno cc_seqno, + std::vector prev_blocks, BlockCandidatePriority priority, + bool is_optimistic, td::Timestamp timeout, td::Promise promise); void process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise); bool can_collate_shard(ShardIdFull shard) const; @@ -58,28 +60,13 @@ class CollatorNode : public td::actor::Actor { std::vector collating_shards_; std::set validator_adnl_ids_; - struct CacheEntry { - bool started = false; - td::Timestamp has_internal_query_at; - td::Timestamp has_external_query_at; - td::Timestamp has_result_at; - BlockSeqno block_seqno = 0; - td::optional result; - td::CancellationTokenSource cancellation_token_source; - std::vector> promises; - - void cancel(td::Status reason); - }; struct ValidatorGroupInfo { CatchainSeqno cc_seqno{0}; std::vector prev; - BlockSeqno next_block_seqno{0}; - std::map, std::shared_ptr> cache; - - void cleanup(); + td::actor::ActorOwn actor; }; struct FutureValidatorGroup { - std::vector> pending_blocks; + std::vector pending_blocks; std::vector> promises; }; std::map validator_groups_; @@ -93,18 +80,17 @@ class CollatorNode : public td::actor::Actor { td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); - void generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - std::optional o_priority, td::Timestamp timeout, - td::Promise promise); - void process_result(std::shared_ptr cache_entry, td::Result R); - td::Status check_out_of_sync(); td::Status check_mc_config(); + bool can_generate() { + return check_out_of_sync().is_ok() && mc_config_status_.is_ok(); + } + + static constexpr int COLLATOR_NODE_VERSION = 1; + public: - static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); - static td::Result deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size, int proto_version); + static constexpr int VERSION_OPTIMISTIC_COLLATE = 1; }; } // namespace ton::validator diff --git a/validator/collator-node/utils.cpp b/validator/collator-node/utils.cpp new file mode 100644 index 000000000..23e2ec40f --- /dev/null +++ b/validator/collator-node/utils.cpp @@ -0,0 +1,93 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "utils.hpp" +#include "checksum.h" +#include "keys/keys.hpp" +#include "ton/ton-tl.hpp" +#include "validator-session/candidate-serializer.h" + +namespace ton::validator { + +tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress) { + if (!compress) { + return create_tl_object( + PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), block.data.clone(), + block.collated_data.clone()); + } + size_t decompressed_size; + td::BufferSlice compressed = + validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok(); + return create_tl_object( + 0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), + (int)decompressed_size, std::move(compressed)); +} + +td::Result deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size, int proto_version) { + td::Result res; + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::collatorNode_candidate& c) { + res = [&]() -> td::Result { + auto hash = td::sha256_bits256(c.collated_data_); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), + std::move(c.collated_data_)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidate& c) { + res = [&]() -> td::Result { + if (c.decompressed_size_ <= 0) { + return td::Status::Error("invalid decompressed size"); + } + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidateV2& c) { + res = [&]() -> td::Result { + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, true, 0, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + })); + return res; +} + +} // namespace ton::validator diff --git a/validator/collator-node/utils.hpp b/validator/collator-node/utils.hpp new file mode 100644 index 000000000..26b67d5a9 --- /dev/null +++ b/validator/collator-node/utils.hpp @@ -0,0 +1,27 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ton/ton-types.h" +#include "tl/generate/auto/tl/ton_api.h" + +namespace ton::validator { + +tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); +td::Result deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size, int proto_version); +} // namespace ton::validator diff --git a/validator/db/package.cpp b/validator/db/package.cpp index 6efcf9c03..959aeeb4a 100644 --- a/validator/db/package.cpp +++ b/validator/db/package.cpp @@ -48,8 +48,16 @@ Package::Package(td::FileFd fd) : fd_(std::move(fd)) { } td::Status Package::truncate(td::uint64 size) { - TRY_STATUS(fd_.seek(size + header_size())); - return fd_.truncate_to_current_position(size + header_size()); + auto target_size = size + header_size(); + TRY_RESULT(current_size, fd_.get_size()); + + // Only truncate if the size actually differs to avoid updating mtime unnecessarily + if (current_size != target_size) { + TRY_STATUS(fd_.seek(target_size)); + return fd_.truncate_to_current_position(target_size); + } + + return td::Status::OK(); } td::uint64 Package::append(std::string filename, td::Slice data, bool sync) { diff --git a/validator/downloaders/wait-block-state-merge.cpp b/validator/downloaders/wait-block-state-merge.cpp index 2b1961610..c5f1cfa7f 100644 --- a/validator/downloaders/wait-block-state-merge.cpp +++ b/validator/downloaders/wait-block-state-merge.cpp @@ -55,7 +55,7 @@ void WaitBlockStateMerge::start_up() { } }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, left_, priority_, timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, left_, priority_, timeout_, false, std::move(P_l)); auto P_r = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -66,7 +66,7 @@ void WaitBlockStateMerge::start_up() { } }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, right_, priority_, timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, right_, priority_, timeout_, false, std::move(P_r)); } diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 2072fe84b..7a2b5a349 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -32,7 +32,11 @@ void WaitBlockState::alarm() { } void WaitBlockState::abort_query(td::Status reason) { - if (promise_) { + if (promise_no_store_) { + promise_no_store_.set_error( + reason.clone().move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); + } + if (promise_final_) { if (priority_ > 0 || (reason.code() != ErrorCode::timeout && reason.code() != ErrorCode::notready)) { LOG(WARNING) << "aborting wait block state query for " << handle_->id() << " priority=" << priority_ << ": " << reason; @@ -40,18 +44,19 @@ void WaitBlockState::abort_query(td::Status reason) { LOG(DEBUG) << "aborting wait block state query for " << handle_->id() << " priority=" << priority_ << ": " << reason; } - promise_.set_error(reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); + promise_final_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); } stop(); } void WaitBlockState::finish_query() { CHECK(handle_->received_state()); - /*if (handle_->id().is_masterchain() && handle_->inited_proof()) { - td::actor::send_closure(manager_, &ValidatorManager::new_block, handle_, prev_state_, [](td::Unit) {}); - }*/ - if (promise_) { - promise_.set_result(prev_state_); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + } + if (promise_final_) { + promise_final_.set_result(prev_state_); } stop(); } @@ -282,10 +287,16 @@ void WaitBlockState::got_block_data(td::Ref data) { } void WaitBlockState::apply() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockState::abort_query, R.move_as_error_prefix("db set error: ")); + } else { + td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); + } + }); + if (opts_->get_permanent_celldb()) { - td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data, handle_, block_, - std::move(promise_)); - stop(); + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data, handle_, block_, std::move(P)); return; } TD_PERF_COUNTER(apply_block_to_state); @@ -296,15 +307,11 @@ void WaitBlockState::apply() { return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &WaitBlockState::abort_query, R.move_as_error_prefix("db set error: ")); - } else { - td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); - } - }); - td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, prev_state_, std::move(P)); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + promise_no_store_ = {}; + } } void WaitBlockState::written_state(td::Ref upd_state) { @@ -324,6 +331,10 @@ void WaitBlockState::got_state_from_db(td::Ref state) { }); td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, prev_state_, std::move(P)); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + promise_no_store_ = {}; + } } else { finish_query(); } diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index a9317381c..cf556e33d 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -28,7 +28,8 @@ class WaitBlockState : public td::actor::Actor { public: WaitBlockState(BlockHandle handle, td::uint32 priority, td::Ref opts, td::Ref last_masterchain_state, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise, + td::Timestamp timeout, td::Promise> promise_no_store, + td::Promise> promise_final, td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) @@ -36,7 +37,8 @@ class WaitBlockState : public td::actor::Actor { , last_masterchain_state_(last_masterchain_state) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) + , promise_no_store_(std::move(promise_no_store)) + , promise_final_(std::move(promise_final)) , persistent_state_desc_(std::move(persistent_state_desc)) , perf_timer_("waitstate", 1.0, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); @@ -96,7 +98,8 @@ class WaitBlockState : public td::actor::Actor { td::Ref last_masterchain_state_; td::actor::ActorId manager_; td::Timestamp timeout_; - td::Promise> promise_; + td::Promise> promise_no_store_; + td::Promise> promise_final_; td::Ref persistent_state_desc_; td::Ref prev_state_; diff --git a/validator/fabric.h b/validator/fabric.h index f49e306ea..7fee48371 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -26,8 +26,33 @@ namespace ton { namespace validator { -enum ValidateMode { fake = 1 }; -enum CollateMode { skip_store_candidate = 1, from_collator_node = 2 }; +struct CollateParams { + ShardIdFull shard; + BlockIdExt min_masterchain_block_id; + std::vector prev; + bool is_hardfork = false; + Ed25519_PublicKey creator{td::Bits256::zero()}; + td::Ref validator_set = {}; + td::Ref collator_opts = {}; + adnl::AdnlNodeIdShort collator_node_id = adnl::AdnlNodeIdShort::zero(); + bool skip_store_candidate = false; + int attempt_idx = 0; + + // Optional - used for optimistic collation + Ref optimistic_prev_block = {}; +}; + +struct ValidateParams { + ShardIdFull shard; + BlockIdExt min_masterchain_block_id; + std::vector prev; + td::Ref validator_set = {}; + PublicKeyHash local_validator_id = PublicKeyHash::zero();; + bool is_fake = false; + + // Optional - used for validation of optimistic candidates + Ref optimistic_prev_block = {}; +}; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, td::Ref opts, bool read_only = false); @@ -80,19 +105,10 @@ void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::Actor td::Ref rel_key_block_proof, bool skip_check_signatures = false); void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); -void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode = 0); -void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, - td::Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, - adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, - int attempt_idx = 0); -void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); +void run_validate_query(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise); +void run_collate_query(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index fa2e03b2e..e9927518a 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -35,6 +35,10 @@ void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonN process_block_broadcast(src, query); } +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); if (B.is_error()) { @@ -94,6 +98,11 @@ void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodeFastSyncOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { BlockIdExt block_id; CatchainSeqno cc_seqno; @@ -316,6 +325,21 @@ void FullNodeFastSyncOverlay::init() { std::make_unique(actor_id(this)), rules, std::move(scope), options); inited_ = true; + if (shard_.is_masterchain()) { + class TelemetryCallback : public ValidatorTelemetry::Callback { + public: + explicit TelemetryCallback(td::actor::ActorId id) : id_(id) { + } + void send_telemetry(tl_object_ptr telemetry) override { + td::actor::send_closure(id_, &FullNodeFastSyncOverlay::send_validator_telemetry, std::move(telemetry)); + } + + private: + td::actor::ActorId id_; + }; + telemetry_sender_ = td::actor::create_actor( + "telemetry", local_id_, std::make_unique(actor_id(this))); + } } void FullNodeFastSyncOverlay::tear_down() { diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index e0db87b7f..3505ae5eb 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -17,6 +17,8 @@ #pragma once #include "full-node.h" +#include "validator-telemetry.hpp" + #include namespace ton::validator::fullnode { @@ -25,6 +27,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast& query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); @@ -32,6 +35,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); void process_telemetry_broadcast(adnl::AdnlNodeIdShort src, @@ -106,6 +110,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void init(); void get_stats_extra(td::Promise promise); + td::actor::ActorOwn telemetry_sender_; bool collect_telemetry_ = false; std::ofstream telemetry_file_; }; diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 68c4b3235..7ea813a46 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -34,6 +34,11 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, process_block_broadcast(src, query); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodePrivateBlockOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); if (B.is_error()) { @@ -63,6 +68,11 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { BlockIdExt block_id; @@ -89,7 +99,7 @@ void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHas } void FullNodePrivateBlockOverlay::process_telemetry_broadcast( - PublicKeyHash src, const tl_object_ptr& telemetry) { + PublicKeyHash src, const tl_object_ptr &telemetry) { if (telemetry->adnl_id_ != src.bits256_value()) { VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": adnl_id mismatch"; return; @@ -107,12 +117,7 @@ void FullNodePrivateBlockOverlay::process_telemetry_broadcast( } VLOG(FULL_NODE_DEBUG) << "Got telemetry broadcast from " << src; auto s = td::json_encode(td::ToJson(*telemetry), false); - s.erase( - std::remove_if(s.begin(), s.end(), [](char c) { - return c == '\n' || c == '\r'; - }), - s.end() - ); + std::erase_if(s, [](char c) { return c == '\n' || c == '\r'; }); telemetry_file_ << s << "\n"; telemetry_file_.flush(); if (telemetry_file_.fail()) { @@ -134,9 +139,7 @@ void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::Buffe } return; } - ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto& obj) { - Self->process_broadcast(src, obj); - }); + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); } void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, @@ -279,6 +282,20 @@ void FullNodePrivateBlockOverlay::init() { td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); inited_ = true; + + class TelemetryCallback : public ValidatorTelemetry::Callback { + public: + explicit TelemetryCallback(td::actor::ActorId id) : id_(id) { + } + void send_telemetry(tl_object_ptr telemetry) override { + td::actor::send_closure(id_, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); + } + + private: + td::actor::ActorId id_; + }; + telemetry_sender_ = td::actor::create_actor("telemetry", local_id_, + std::make_unique(actor_id(this))); } void FullNodePrivateBlockOverlay::tear_down() { @@ -295,6 +312,10 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNod process_block_broadcast(src, query); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { VLOG(FULL_NODE_DEBUG) << "Dropping block broadcast in private overlay \"" << name_ << "\" from unauthorized sender " @@ -333,6 +354,11 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 70e196ea6..2cbe25f68 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -17,6 +17,8 @@ #pragma once #include "full-node.h" +#include "validator-telemetry.hpp" + #include namespace ton::validator::fullnode { @@ -25,12 +27,14 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_telemetry_broadcast(PublicKeyHash src, const tl_object_ptr& telemetry); @@ -98,6 +102,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void try_init(); void init(); + td::actor::ActorOwn telemetry_sender_; bool collect_telemetry_ = false; std::ofstream telemetry_file_; }; @@ -106,12 +111,14 @@ class FullNodeCustomOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); template diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp index 94dc2155e..5665cc233 100644 --- a/validator/full-node-serializer.cpp +++ b/validator/full-node-serializer.cpp @@ -20,6 +20,7 @@ #include "auto/tl/ton_api.hpp" #include "tl-utils/tl-utils.hpp" #include "vm/boc.h" +#include "vm/boc-compression.h" #include "td/utils/lz4.h" #include "full-node.h" #include "td/utils/overloaded.h" @@ -88,6 +89,28 @@ static td::Result deserialize_block_broadcast(ton_api::tonNode_b std::move(proof)}; } +static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcastCompressedV2& f, + int max_decompressed_size) { + std::vector signatures; + for (auto& sig : f.signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } + TRY_RESULT(roots, vm::boc_decompress(f.compressed_, max_decompressed_size)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block broadcast: " << f.compressed_.size() << " -> " + << data.size() + proof.size() + signatures.size() * 96; + return BlockBroadcast{create_block_id(f.id_), + std::move(signatures), + static_cast(f.catchain_seqno_), + static_cast(f.validator_set_hash_), + std::move(data), + std::move(proof)}; +} + td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size) { td::Result B; @@ -96,6 +119,9 @@ td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcas [&](ton_api::tonNode_blockBroadcastCompressed& f) { B = deserialize_block_broadcast(f, max_decompressed_data_size); }, + [&](ton_api::tonNode_blockBroadcastCompressedV2& f) { + B = deserialize_block_broadcast(f, max_decompressed_data_size); + }, [&](auto&) { B = td::Status::Error("unknown broadcast type"); })); return B; } @@ -139,6 +165,20 @@ static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, return td::Status::OK(); } +static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressedV2& f, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) { + TRY_RESULT(roots, vm::boc_decompress(f.compressed_, max_decompressed_size)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block full V2: " << f.compressed_.size() << " -> " << data.size() + proof.size(); + id = create_block_id(f.id_); + is_proof_link = f.is_link_; + return td::Status::OK(); +} + td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof, td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size) { td::Status S; @@ -148,6 +188,9 @@ td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id [&](ton_api::tonNode_dataFullCompressed& f) { S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size); }, + [&](ton_api::tonNode_dataFullCompressedV2& f) { + S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size); + }, [&](auto&) { S = td::Status::Error("unknown data type"); })); return S; } @@ -194,6 +237,24 @@ static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBloc return td::Status::OK(); } +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data, + int max_decompressed_data_size) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + TRY_RESULT(roots, vm::boc_decompress(obj.compressed_, max_decompressed_data_size)); + if (roots.size() != 1) { + return td::Status::Error("expected 1 root in boc"); + } + auto root = std::move(roots[0]); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(root, 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block candidate broadcast V2: " << obj.compressed_.size() << " -> " + << data.size(); + return td::Status::OK(); +} + td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, td::BufferSlice& data, int max_decompressed_data_size) { @@ -207,6 +268,10 @@ td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, data, max_decompressed_data_size); }, + [&](ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data, max_decompressed_data_size); + }, [&](auto&) { S = td::Status::Error("unknown data type"); })); return S; } diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index ce33f716d..a175d79ac 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -804,6 +804,11 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodeShardImpl::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { BlockIdExt block_id; CatchainSeqno cc_seqno; @@ -832,6 +837,10 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl process_block_broadcast(src, query); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); if (B.is_error()) { diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 534693d95..a90460adb 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -152,6 +152,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); @@ -163,6 +164,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index b9ecb1666..436cbacac 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -318,21 +318,28 @@ void FullNodeImpl::send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice dat } void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) { - auto shard = get_shard(dst); - if (shard.empty()) { - VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; - return; - } + bool skip_public = false; for (auto &[_, private_overlay] : custom_overlays_) { if (private_overlay.params_.send_shard(dst.as_leaf_shard())) { for (auto &[local_id, actor] : private_overlay.actors_) { if (private_overlay.params_.msg_senders_.find(local_id) != private_overlay.params_.msg_senders_.end()) { td::actor::send_closure(actor, &FullNodeCustomOverlay::send_external_message, data.clone()); + if (private_overlay.params_.skip_public_msg_send_) { + skip_public = true; + } } } } } - td::actor::send_closure(shard, &FullNodeShard::send_external_message, std::move(data)); + + if (!skip_public) { + auto shard = get_shard(dst); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; + return; + } + td::actor::send_closure(shard, &FullNodeShard::send_external_message, std::move(data)); + } } void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { @@ -437,7 +444,7 @@ void FullNodeImpl::download_zero_state(BlockIdExt id, td::uint32 priority, td::T void FullNodeImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(id.shard_full()); + auto shard = get_shard(id.shard_full(), /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download state diff query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); @@ -460,7 +467,7 @@ void FullNodeImpl::download_block_proof(BlockIdExt block_id, td::uint32 priority void FullNodeImpl::download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(block_id.shard_full()); + auto shard = get_shard(block_id.shard_full(), /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download proof link query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); @@ -483,7 +490,7 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(shard_prefix); + auto shard = get_shard(shard_prefix, /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download archive query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); @@ -512,7 +519,7 @@ void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vect timeout, std::move(promise)); } -td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { +td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, bool historical) { if (shard.is_masterchain()) { return shards_[ShardIdFull{masterchainId}].actor.get(); } @@ -520,8 +527,12 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { return {}; } int pfx_len = shard.pfx_len(); - if (pfx_len > wc_monitor_min_split_) { - shard = shard_prefix(shard, wc_monitor_min_split_); + int min_split = wc_monitor_min_split_; + if (historical) { + min_split = td::Random::fast(0, min_split); + } + if (pfx_len > min_split) { + shard = shard_prefix(shard, min_split); } while (true) { auto it = shards_.find(shard); @@ -614,24 +625,6 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { } } -void FullNodeImpl::send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) { - if (use_old_private_overlays_) { - auto it = private_block_overlays_.find(key); - if (it == private_block_overlays_.end()) { - VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for " << key << " : no private block overlay"; - return; - } - td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); - } else { - auto overlay = fast_sync_overlays_.get_masterchain_overlay_for(adnl::AdnlNodeIdShort{telemetry->adnl_id_}); - if (overlay.empty()) { - VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for adnl id " << key << " : no fast sync overlay"; - return; - } - td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::send_validator_telemetry, std::move(telemetry)); - } -} - void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { send_block_broadcast_to_custom_overlays(broadcast); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_broadcast, std::move(broadcast), @@ -777,9 +770,6 @@ void FullNodeImpl::start_up() { void new_key_block(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - td::actor::send_closure(id_, &FullNodeImpl::send_validator_telemetry, key, std::move(telemetry)); - } explicit Callback(td::actor::ActorId id) : id_(id) { } @@ -964,6 +954,7 @@ CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_c for (const auto &shard : f.sender_shards_) { c.sender_shards_.push_back(create_shard_id(shard)); } + c.skip_public_msg_send_ = f.skip_public_msg_send_; return c; } diff --git a/validator/full-node.h b/validator/full-node.h index dc29f67ca..4c11c1fbd 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -67,6 +67,7 @@ struct CustomOverlayParams { std::map msg_senders_; std::set block_senders_; std::vector sender_shards_; + bool skip_public_msg_send_ = false; bool send_shard(const ShardIdFull& shard) const; static CustomOverlayParams fetch(const ton_api::engine_validator_customOverlay& f); diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 9a70d5c0c..482a916bb 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -90,7 +90,6 @@ class FullNodeImpl : public FullNode { void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry); void process_block_broadcast(BlockBroadcast broadcast) override; void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, @@ -131,7 +130,7 @@ class FullNodeImpl : public FullNode { FileHash zero_state_file_hash_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); - td::actor::ActorId get_shard(ShardIdFull shard); + td::actor::ActorId get_shard(ShardIdFull shard, bool historical = false); std::map shards_; int wc_monitor_min_split_ = 0; diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index cbb03bac1..883994089 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -494,6 +494,17 @@ void AcceptBlockQuery::written_block_signatures() { void AcceptBlockQuery::written_block_info() { VLOG(VALIDATOR_DEBUG) << "written block info"; if (data_.not_null()) { + block_root_ = data_->root_cell(); + if (block_root_.is_null()) { + fatal_error("block data does not contain a root cell"); + return; + } + // generate proof + if (!create_new_proof()) { + fatal_error("cannot generate proof for block "s + id_.to_str()); + return; + } + send_broadcasts(); if (!apply_) { written_state({}); return; @@ -563,17 +574,6 @@ void AcceptBlockQuery::written_state(td::Ref upd_state) { CHECK(data_.not_null()); state_ = std::move(upd_state); - block_root_ = data_->root_cell(); - if (block_root_.is_null()) { - fatal_error("block data does not contain a root cell"); - return; - } - // generate proof - if (!create_new_proof()) { - fatal_error("cannot generate proof for block "s + id_.to_str()); - return; - } - if (apply_ && state_keep_old_hash_ != state_old_hash_) { fatal_error(PSTRING() << "invalid previous state hash in newly-created proof: expected " << state_->root_hash().to_hex() << ", found in update " << state_old_hash_.to_hex()); @@ -626,7 +626,7 @@ void AcceptBlockQuery::got_last_mc_block(std::pair, Bl VLOG(VALIDATOR_DEBUG) << "shardchain block refers to newer masterchain block " << mc_blkid_.to_str() << ", trying to obtain it"; td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(), timeout_, - [SelfId = actor_id(this)](td::Result> R) { + false, [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_mc_state, R.move_as_ok()); @@ -935,8 +935,11 @@ void AcceptBlockQuery::written_block_info_2() { } void AcceptBlockQuery::applied() { + finish_query(); +} + +void AcceptBlockQuery::send_broadcasts() { if (send_broadcast_mode_ == 0) { - finish_query(); return; } BlockBroadcast b; @@ -964,8 +967,6 @@ void AcceptBlockQuery::applied() { // td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id_, // validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), // std::move(b.data), send_broadcast_mode_); - - finish_query(); } } // namespace validator diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index d8b763ba0..e582650c1 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -89,6 +89,7 @@ class AcceptBlockQuery : public td::actor::Actor { void written_block_next(); void written_block_info_2(); void applied(); + void send_broadcasts(); private: BlockIdExt id_; diff --git a/validator/impl/check-proof.cpp b/validator/impl/check-proof.cpp index 30a13c089..12d336ad1 100644 --- a/validator/impl/check-proof.cpp +++ b/validator/impl/check-proof.cpp @@ -344,7 +344,7 @@ void CheckProof::got_block_handle(BlockHandle handle) { process_masterchain_state(); return; } - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, prev_[0], priority(), timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, prev_[0], priority(), timeout_, false, [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &CheckProof::got_masterchain_state, diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index fcc95caad..83ae4370a 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "block-parse.h" #include "interfaces/validator-manager.h" #include "shard.hpp" #include "top-shard-descr.hpp" @@ -33,6 +34,7 @@ #include #include #include "common/global-version.h" +#include "fabric.h" namespace ton { @@ -80,7 +82,8 @@ class Collator final : public td::actor::Actor { td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; adnl::AdnlNodeIdShort collator_node_id_ = adnl::AdnlNodeIdShort::zero(); - unsigned mode_ = 0; + bool skip_store_candidate_ = false; + Ref optimistic_prev_block_; int attempt_idx_; bool allow_repeat_collation_ = false; ton::BlockSeqno last_block_seqno{0}; @@ -95,11 +98,8 @@ class Collator final : public td::actor::Actor { static constexpr bool shard_splitting_enabled = true; public: - Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, - Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, - td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, - int attempt_idx); + Collator(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise); ~Collator() override = default; bool is_busy() const { return busy_; @@ -118,10 +118,12 @@ class Collator final : public td::actor::Actor { Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, block::ActionPhaseConfig* action_phase_cfg, block::SerializeConfig* serialize_cfg, bool external, - LogicalTime after_lt); + LogicalTime after_lt, CollationStats* stats = nullptr); private: void start_up() override; + void load_prev_states_blocks(); + bool process_optimistic_prev_block(); void alarm() override; int verbosity{3 * 0}; int verify{1}; @@ -156,6 +158,7 @@ class Collator final : public td::actor::Actor { ton::LogicalTime shards_max_end_lt_{0}; ton::UnixTime prev_state_utime_; int global_id_{0}; + int global_version_{0}; ton::BlockSeqno min_ref_mc_seqno_{~0U}; ton::BlockSeqno vert_seqno_{~0U}, prev_vert_seqno_{~0U}; ton::BlockIdExt prev_key_block_; @@ -203,6 +206,8 @@ class Collator final : public td::actor::Actor { std::vector ext_msg_list_; std::priority_queue, std::greater> new_msgs; std::pair last_proc_int_msg_, first_unproc_int_msg_; + block::tlb::Aug_InMsgDescr aug_InMsgDescr{0}; + block::tlb::Aug_OutMsgDescr aug_OutMsgDescr{0}; std::unique_ptr in_msg_dict, out_msg_dict, old_out_msg_queue_, out_msg_queue_, sibling_out_msg_queue_; std::map unprocessed_deferred_messages_; // number of messages from dispatch queue in new_msgs @@ -276,6 +281,7 @@ class Collator final : public td::actor::Actor { void after_get_shard_blocks(td::Result>> res, td::PerfLogAction token); void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, td::PerfLogAction token); + void after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token); bool preprocess_prev_mc_state(); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); @@ -326,6 +332,14 @@ class Collator final : public td::actor::Actor { bool is_masterchain() const { return shard_.is_masterchain(); } + int prev_block_idx(const BlockIdExt& id) const { + for (size_t i = 0; i < prev_blocks.size(); ++i) { + if (prev_blocks[i] == id) { + return (int)i; + } + } + return -1; + } bool is_our_address(Ref addr_ref) const; bool is_our_address(ton::AccountIdPrefixFull addr_prefix) const; bool is_our_address(const ton::StdSmcAddress& addr) const; @@ -394,7 +408,7 @@ class Collator final : public td::actor::Actor { bool create_collated_data(); bool create_block_candidate(); - void return_block_candidate(td::Result saved); + void return_block_candidate(td::Result saved, td::PerfLogAction token); bool update_last_proc_int_msg(const std::pair& new_lt_hash); td::CancellationToken cancellation_token_; @@ -404,8 +418,7 @@ class Collator final : public td::actor::Actor { static td::uint32 get_skip_externals_queue_size(); private: - td::Timer work_timer_{true}; - td::ThreadCpuTimer cpu_work_timer_{true}; + td::RealCpuTimer work_timer_{true}; CollationStats stats_; void finalize_stats(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index acf9075b4..14c2d9aed 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -58,33 +58,21 @@ static constexpr int MAX_ATTEMPTS = 5; /** * Constructs a Collator object. * - * @param shard The shard of the new block. - * @param is_hardfork A boolean indicating whether the new block is a hardfork. - * @param min_masterchain_block_id The the minimum reference masterchain block. - * @param prev A vector of BlockIdExt representing the previous blocks. - * @param validator_set A reference to the ValidatorSet. - * @param collator_id The public key of the block creator. - * @param collator_opts A reference to CollatorOptions. + * @param params Collator parameters * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the collator. - * @param promise The promise to return the result. - * @param collator_node_id ADNL id of the collator node that generates the block (zero if it's not a collator node) * @param cancellation_token Token to cancel collation. - * @param mode +1 - skip storing candidate to disk, +2 - called from CollatorNode. - * @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps. - */ -Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, - std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, - Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, adnl::AdnlNodeIdShort collator_node_id, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) - : shard_(shard) - , is_hardfork_(is_hardfork) - , min_mc_block_id{min_masterchain_block_id} - , prev_blocks(std::move(prev)) - , created_by_(collator_id) - , collator_opts_(collator_opts) - , validator_set_(std::move(validator_set)) + * @param promise The promise to return the result. + */ +Collator::Collator(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise) + : shard_(params.shard) + , is_hardfork_(params.is_hardfork) + , min_mc_block_id{params.min_masterchain_block_id} + , prev_blocks(std::move(params.prev)) + , created_by_(params.creator) + , collator_opts_(params.collator_opts.is_null() ? td::Ref{true} : params.collator_opts) + , validator_set_(std::move(params.validator_set)) , manager(manager) , timeout(timeout) // default timeout is 10 seconds, declared in validator/validator-group.cpp:generate_block_candidate:run_collate_query @@ -92,9 +80,10 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) - , collator_node_id_(collator_node_id) - , mode_(mode) - , attempt_idx_(attempt_idx) + , collator_node_id_(params.collator_node_id) + , skip_store_candidate_(params.skip_store_candidate) + , optimistic_prev_block_(std::move(params.optimistic_prev_block)) + , attempt_idx_(params.attempt_idx) , perf_timer_("collate", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); @@ -204,6 +193,16 @@ void Collator::start_up() { return; } } + if (optimistic_prev_block_.not_null()) { + if (prev_blocks.size() != 1) { + fatal_error(-666, "optimistic prev block is not null, which is not allowed after merge"); + return; + } + if (prev_blocks[0] != optimistic_prev_block_->block_id()) { + fatal_error(-666, "optimistic prev block is not null, but has invalid block id"); + return; + } + } busy_ = true; step = 1; if (!is_masterchain()) { @@ -239,31 +238,11 @@ void Collator::start_up() { // 3. load previous block(s) and corresponding state(s) prev_states.resize(prev_blocks.size()); prev_block_data.resize(prev_blocks.size()); - for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 3.1. load state - LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - auto token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i); - td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, - [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { - LOG(DEBUG) << "got answer to wait_block_state query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, std::move(res), - std::move(token)); - }); - if (prev_blocks[i].seqno()) { - // 3.2. load block - // NB: we need the block itself only for extracting start_lt and end_lt to create correct prev_blk:ExtBlkRef and related Merkle proofs - LOG(DEBUG) << "sending wait_block_data() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - auto token = perf_log_.start_action(PSTRING() << "wait_block_data #" << i); - td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), timeout, - [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { - LOG(DEBUG) << "got answer to wait_block_data query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, std::move(res), - std::move(token)); - }); + if (optimistic_prev_block_.is_null()) { + load_prev_states_blocks(); + } else { + if (!process_optimistic_prev_block()) { + return; } } if (is_hardfork_) { @@ -312,6 +291,85 @@ void Collator::start_up() { CHECK(pending); } +/** + * Load previous states and blocks from DB + */ +void Collator::load_prev_states_blocks() { + for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { + // 3.1. load state + LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + auto token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, false, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, std::move(res), + std::move(token)); + }); + if (prev_blocks[i].seqno()) { + // 3.2. load block + LOG(DEBUG) << "sending wait_block_data() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + auto token = perf_log_.start_action(PSTRING() << "wait_block_data #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), timeout, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_data query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, std::move(res), + std::move(token)); + }); + } + } +} + +/** + * Write optimistic prev block as block data, load previous state to apply Merkle update to it + */ +bool Collator::process_optimistic_prev_block() { + std::vector prev_prev; + BlockIdExt mc_blkid; + bool after_split; + auto S = block::unpack_block_prev_blk_try(optimistic_prev_block_->root_cell(), optimistic_prev_block_->block_id(), + prev_prev, mc_blkid, after_split); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); + } + // 3.1. load state + if (prev_prev.size() == 1) { + LOG(DEBUG) << "sending wait_block_state() query for " << prev_prev[0].to_str() << " to Manager (opt)"; + ++pending; + auto token = perf_log_.start_action("opt wait_block_state"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, false, + [self = get_self(), token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query (opt)"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state_optimistic, std::move(res), + std::move(token)); + }); + } else { + CHECK(prev_prev.size() == 2); + LOG(DEBUG) << "sending wait_block_state_merge() query for " << prev_prev[0].to_str() << " and " + << prev_prev[1].to_str() << " to Manager (opt)"; + ++pending; + auto token = perf_log_.start_action("opt wait_block_state_merge"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], priority(), timeout, + [self = get_self(), token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state_optimistic, std::move(res), + std::move(token)); + }); + } + // 3.2. load block + LOG(DEBUG) << "use optimistic prev block " << prev_blocks[0].to_str(); + ++pending; + auto token = perf_log_.start_action(PSTRING() << "opt wait_block_data"); + td::actor::send_closure_later(actor_id(this), &Collator::after_get_block_data, 0, optimistic_prev_block_, + std::move(token)); + return true; +} + /** * Raises an error when timeout is reached. */ @@ -377,9 +435,18 @@ bool Collator::fatal_error(td::Status error) { if (allow_repeat_collation_ && error.code() != ErrorCode::cancelled && attempt_idx_ + 1 < MAX_ATTEMPTS && !is_hardfork_ && !timeout.is_in_past()) { LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")"; - run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager, - td::Timestamp::in(10.0), std::move(main_promise), collator_node_id_, - std::move(cancellation_token_), mode_, attempt_idx_ + 1); + run_collate_query(CollateParams{.shard = shard_, + .min_masterchain_block_id = min_mc_block_id, + .prev = prev_blocks, + .is_hardfork = false, + .creator = created_by_, + .validator_set = validator_set_, + .collator_opts = collator_opts_, + .collator_node_id = collator_node_id_, + .skip_store_candidate = skip_store_candidate_, + .attempt_idx = attempt_idx_ + 1, + .optimistic_prev_block = optimistic_prev_block_}, + manager, td::Timestamp::in(10.0), std::move(cancellation_token_), std::move(main_promise)); } else { LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error; LOG(INFO) << perf_log_; @@ -506,9 +573,9 @@ bool Collator::request_aux_mc_state(BlockSeqno seqno, Ref& st CHECK(blkid.is_valid_ext() && blkid.is_masterchain()); LOG(DEBUG) << "sending auxiliary wait_block_state() query for " << blkid.to_str() << " to Manager"; ++pending; - auto token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.to_str()); + auto token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.seqno()); td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, + manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, false, [self = get_self(), blkid, token = std::move(token)](td::Result> res) mutable { LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); td::actor::send_closure_later(std::move(self), &Collator::after_get_aux_shard_state, blkid, std::move(res), @@ -678,7 +745,7 @@ void Collator::after_get_shard_state(int idx, td::Result> res, t * Callback function called after retrieving block data for a previous block. * * @param idx The index of the previous block (0 or 1). - * @param res The retreved block data. + * @param res The retrieved block data. */ void Collator::after_get_block_data(int idx, td::Result> res, td::PerfLogAction token) { LOG(DEBUG) << "in Collator::after_get_block_data(" << idx << ")"; @@ -757,6 +824,31 @@ void Collator::after_get_storage_stat_cache(td::Result> res, td::PerfLogAction token) { + LOG(DEBUG) << "in Collator::after_get_shard_state_optimistic()"; + token.finish(res); + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + td::RealCpuTimer timer; + work_timer_.resume(); + auto state = res.move_as_ok(); + auto S = state.write().apply_block(optimistic_prev_block_->block_id(), optimistic_prev_block_); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("apply error: ")); + return; + } + work_timer_.pause(); + stats_.work_time.optimistic_apply = timer.elapsed_both(); + after_get_shard_state(0, std::move(state), {}); +} + /** * Unpacks the last masterchain state and initializes the Collator object with the extracted configuration. * @@ -764,7 +856,7 @@ void Collator::after_get_storage_stat_cache(td::Resultset_block_id_ext(mc_block_id_); global_id_ = config_->get_global_blockchain_id(); + global_version_ = config_->get_global_version(); ihr_enabled_ = config_->ihr_enabled(); create_stats_enabled_ = config_->create_stats_enabled(); report_version_ = config_->has_capability(ton::capReportVersion); @@ -896,7 +988,9 @@ bool Collator::request_neighbor_msg_queues() { unsigned i = 0; for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); - top_blocks.push_back(descr.blk_); + if (prev_block_idx(descr.blk_) == -1) { + top_blocks.push_back(descr.blk_); + } ++i; } ++pending; @@ -935,8 +1029,7 @@ bool Collator::request_out_msg_queue_size() { /** * Handles the result of obtaining the outbound queue for a neighbor. * - * @param i The index of the neighbor. - * @param res The obtained outbound queue. + * @param R The result of retrieving neighbor message queues (top block id -> queue). */ void Collator::got_neighbor_msg_queues(td::Result>> R, td::PerfLogAction token) { @@ -952,12 +1045,17 @@ void Collator::got_neighbor_msg_queues(td::Result= 0) { + got_neighbor_msg_queue( + i, Ref{true, descr.blk_, prev_states[prev_idx]->root_cell(), td::Ref{}, true}); + } else { + auto it = res.find(descr.blk_); + if (it == res.end()) { + fatal_error(PSTRING() << "no msg queue from neighbor #" << i); + return; + } + got_neighbor_msg_queue(i, it->second); } - got_neighbor_msg_queue(i, it->second); ++i; } check_pending(); @@ -1963,10 +2061,8 @@ bool Collator::register_shard_block_creators(std::vector creator_li */ bool Collator::try_collate() { work_timer_.resume(); - cpu_work_timer_.resume(); SCOPE_EXIT { work_timer_.pause(); - cpu_work_timer_.pause(); }; if (!preinit_complete) { LOG(WARNING) << "running do_preinit()"; @@ -2292,8 +2388,9 @@ bool Collator::do_collate() { return fatal_error("cannot fetch required configuration parameters from masterchain state"); } LOG(DEBUG) << "config parameters fetched, creating message dictionaries"; - in_msg_dict = std::make_unique(256, block::tlb::aug_InMsgDescr); - out_msg_dict = std::make_unique(256, block::tlb::aug_OutMsgDescr); + aug_InMsgDescr.global_version = aug_OutMsgDescr.global_version = global_version_; + in_msg_dict = std::make_unique(256, aug_InMsgDescr); + out_msg_dict = std::make_unique(256, aug_OutMsgDescr); LOG(DEBUG) << "message dictionaries created"; if (max_lt == start_lt) { ++max_lt; @@ -2450,8 +2547,10 @@ bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime deli * @returns True if the cleanup operation was successful, false otherwise. */ bool Collator::out_msg_queue_cleanup() { + td::RealCpuTimer timer; SCOPE_EXIT { stats_.load_fraction_queue_cleanup = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + stats_.work_time.queue_cleanup = timer.elapsed_both(); }; LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors"; if (verbosity >= 2) { @@ -2677,6 +2776,10 @@ bool Collator::init_account_storage_dict(block::Account& account) { if (storage_dict_hash.is_zero()) { return true; } + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.prelim_storage_stat = timer.elapsed_both(); + }; td::Ref cached_dict_root = storage_stat_cache_ ? storage_stat_cache_(storage_dict_hash) : td::Ref{}; if (cached_dict_root.not_null()) { @@ -2684,6 +2787,14 @@ bool Collator::init_account_storage_dict(block::Account& account) { LOG(DEBUG) << "Inited storage stat from cache for account " << account.addr.to_hex() << " (" << account.storage_used.cells << " cells)"; storage_stat_cache_update_.emplace_back(cached_dict_root, account.storage_used.cells); + stats_.storage_stat_cache.hit_cnt++; + stats_.storage_stat_cache.hit_cells += account.storage_used.cells; + } else if (account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { + stats_.storage_stat_cache.miss_cnt++; + stats_.storage_stat_cache.miss_cells += account.storage_used.cells; + } else { + stats_.storage_stat_cache.small_cnt++; + stats_.storage_stat_cache.small_cells += account.storage_used.cells; } if (!full_collated_data_ || is_masterchain()) { if (cached_dict_root.not_null()) { @@ -2847,6 +2958,10 @@ static td::Ref clean_usage_cells(td::Ref old_root, td::Ref= StorageStatCache::MIN_ACCOUNT_CELLS; @@ -3118,7 +3233,7 @@ bool Collator::create_special_transaction(block::CurrencyCollection amount, Ref< && cb.store_long_bool(0x4ff, 11) // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 && cb.store_bits_bool(addr) // address:bits256 => dest:MsgAddressInt && amount.store(cb) // value:CurrencyCollection - && cb.store_zeroes_bool(4 + 4) // ihr_fee:Grams fwd_fee:Grams + && cb.store_zeroes_bool(4 + 4) // extra_flags:(VarUInteger 16) fwd_fee:Grams && cb.store_long_bool(lt, 64) // created_lt:uint64 && cb.store_long_bool(now_, 32) // created_at:uint32 && cb.store_zeroes_bool(2) // init:(Maybe ...) body:(Either X ^X) = Message X @@ -3188,6 +3303,12 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t std::unique_ptr trans = std::make_unique( *acc, mask == 2 ? block::transaction::Transaction::tr_tick : block::transaction::Transaction::tr_tock, req_start_lt, now_); + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.trx_tvm += trans->time_tvm; + stats_.work_time.trx_storage_stat += trans->time_storage_stat; + stats_.work_time.trx_other += timer.elapsed_both() - trans->time_tvm - trans->time_storage_stat; + }; if (!trans->prepare_storage_phase(storage_phase_cfg_, true)) { return fatal_error(td::Status::Error( -666, std::string{"cannot create storage phase of a new transaction for smart contract "} + smc_addr.to_hex())); @@ -3292,7 +3413,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, } set_current_tx_storage_dict(*acc); auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, &serialize_cfg_, external, after_lt); + &action_phase_cfg_, &serialize_cfg_, external, after_lt, &stats_); if (res.is_error()) { auto error = res.move_as_error(); if (error.code() == -701) { @@ -3348,19 +3469,17 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, * @param serialize_cfg The configuration for the serialization of the transaction. * @param external Flag indicating if the message is external. * @param after_lt The logical time after which the transaction should occur. Used only for external messages. + * @param collation_stats Stats to write real/cpu time to (optional) * * @returns A Result object containing the created transaction. * Returns error_code == 669 if the error is fatal and the block can not be produced. * Returns error_code == 701 if the transaction can not be included into block, but it's ok (external or too early internal). */ -td::Result> Collator::impl_create_ordinary_transaction(Ref msg_root, - block::Account* acc, - UnixTime utime, LogicalTime lt, - block::StoragePhaseConfig* storage_phase_cfg, - block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg, - block::SerializeConfig* serialize_cfg, - bool external, LogicalTime after_lt) { +td::Result> Collator::impl_create_ordinary_transaction( + Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, block::SerializeConfig* serialize_cfg, bool external, + LogicalTime after_lt, CollationStats* stats) { if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() << " is too large"); @@ -3372,64 +3491,74 @@ td::Result> Collator::impl_crea std::unique_ptr trans = std::make_unique( *acc, block::transaction::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); - bool ihr_delivered = false; // FIXME - if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { - if (external) { - // inbound external message was not accepted - return td::Status::Error(-701, "inbound external message rejected by account "s + acc->addr.to_hex() + - " before smart-contract execution"); + { + td::RealCpuTimer timer; + SCOPE_EXIT { + if (stats) { + stats->work_time.trx_tvm += trans->time_tvm; + stats->work_time.trx_storage_stat += trans->time_storage_stat; + stats->work_time.trx_other += timer.elapsed_both() - trans->time_tvm - trans->time_storage_stat; + } + }; + bool ihr_delivered = false; // FIXME + if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { + if (external) { + // inbound external message was not accepted + return td::Status::Error(-701, "inbound external message rejected by account "s + acc->addr.to_hex() + + " before smart-contract execution"); + } + return td::Status::Error(-669, "cannot unpack input message for a new transaction"); } - return td::Status::Error(-669, "cannot unpack input message for a new transaction"); - } - if (trans->bounce_enabled) { - if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { - return td::Status::Error( - -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + if (trans->bounce_enabled) { + if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { + return td::Status::Error( + -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error( + -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } else { + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error( + -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + return td::Status::Error( + -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } } - if (!external && !trans->prepare_credit_phase()) { + if (!trans->prepare_compute_phase(*compute_phase_cfg)) { return td::Status::Error( - -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->compute_phase->accepted) { + if (external) { + // inbound external message was not accepted + auto const& cp = *trans->compute_phase; + return td::Status::Error( + -701, PSLICE() << "inbound external message rejected by transaction " << acc->addr.to_hex() << ":\n" + << "exitcode=" << cp.exit_code << ", steps=" << cp.vm_steps << ", gas_used=" << cp.gas_used + << (cp.vm_log.empty() ? "" : "\nVM Log (truncated):\n..." + cp.vm_log)); + } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { + return td::Status::Error(-669, "new ordinary transaction for smart contract "s + acc->addr.to_hex() + + " has not been accepted by the smart contract (?)"); + } } - } else { - if (!external && !trans->prepare_credit_phase()) { + if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { return td::Status::Error( - -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + if (trans->bounce_enabled && + (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) && + !trans->prepare_bounce_phase(*action_phase_cfg)) { return td::Status::Error( - -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - } - if (!trans->prepare_compute_phase(*compute_phase_cfg)) { - return td::Status::Error( - -669, "cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (!trans->compute_phase->accepted) { - if (external) { - // inbound external message was not accepted - auto const& cp = *trans->compute_phase; - return td::Status::Error( - -701, PSLICE() << "inbound external message rejected by transaction " << acc->addr.to_hex() << ":\n" - << "exitcode=" << cp.exit_code << ", steps=" << cp.vm_steps << ", gas_used=" << cp.gas_used - << (cp.vm_log.empty() ? "" : "\nVM Log (truncated):\n..." + cp.vm_log)); - } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { - return td::Status::Error(-669, "new ordinary transaction for smart contract "s + acc->addr.to_hex() + - " has not been accepted by the smart contract (?)"); + if (!trans->serialize(*serialize_cfg)) { + return td::Status::Error(-669, "cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); } } - if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { - return td::Status::Error( - -669, "cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (trans->bounce_enabled && - (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) && - !trans->prepare_bounce_phase(*action_phase_cfg)) { - return td::Status::Error( - -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (!trans->serialize(*serialize_cfg)) { - return td::Status::Error(-669, "cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); - } return std::move(trans); } @@ -6018,6 +6147,10 @@ bool Collator::create_mc_block_extra(Ref& mc_block_extra) { * @returns True if the new block is successfully created, false otherwise. */ bool Collator::create_block() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_block += timer.elapsed_both(); + }; Ref block_info, extra; if (!create_block_info(block_info)) { return fatal_error("cannot create BlockInfo for the new block"); @@ -6095,7 +6228,7 @@ Ref Collator::collate_shard_block_descr_set() { /** * Visits certain cells in out msg queue and dispatch queue to add them to the proof * - * @returns True on success, Falise if error occurred + * @returns True on success, False if error occurred */ bool Collator::prepare_msg_queue_proof() { auto res = old_out_msg_queue_->scan_diff( @@ -6156,6 +6289,10 @@ bool Collator::prepare_msg_queue_proof() { * @returns True if the collated data was successfully created, false otherwise. */ bool Collator::create_collated_data() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_collated_data += timer.elapsed_both(); + }; // 1. store the set of used shard block descriptions if (!used_shard_block_descr_.empty()) { auto cell = collate_shard_block_descr_set(); @@ -6253,6 +6390,10 @@ bool Collator::create_collated_data() { * @returns True if the block candidate was created successfully, false otherwise. */ bool Collator::create_block_candidate() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_block_candidate += timer.elapsed_both(); + }; auto consensus_config = config_->get_consensus_config(); // 1. serialize block LOG(INFO) << "serializing new Block"; @@ -6351,17 +6492,19 @@ bool Collator::create_block_candidate() { << consensus_config.max_collated_data_size << ")"); } // 4. save block candidate - if (mode_ & CollateMode::skip_store_candidate) { - td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit()); + if (skip_store_candidate_) { + td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit(), td::PerfLogAction{}); } else { LOG(INFO) << "saving new BlockCandidate"; - td::actor::send_closure_later( - manager, &ValidatorManager::set_block_candidate, block_candidate->id, block_candidate->clone(), - validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), - [self = get_self()](td::Result saved) -> void { - LOG(DEBUG) << "got answer to set_block_candidate"; - td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, std::move(saved)); - }); + auto token = perf_log_.start_action("set_block_candidate"); + td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, + block_candidate->clone(), validator_set_->get_catchain_seqno(), + validator_set_->get_validator_set_hash(), + [self = get_self(), token = std::move(token)](td::Result saved) mutable { + LOG(DEBUG) << "got answer to set_block_candidate"; + td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, + std::move(saved), std::move(token)); + }); } // 5. communicate about bad and delayed external messages if (!bad_ext_msgs_.empty() || !delay_ext_msgs_.empty()) { @@ -6381,8 +6524,9 @@ bool Collator::create_block_candidate() { * * @param saved The result of saving the block candidate to the disk. */ -void Collator::return_block_candidate(td::Result saved) { +void Collator::return_block_candidate(td::Result saved, td::PerfLogAction token) { // 6. return data to the original "caller" + token.finish(saved); if (saved.is_error()) { auto err = saved.move_as_error(); LOG(ERROR) << "cannot save block candidate: " << err.to_string(); @@ -6524,28 +6668,27 @@ td::uint32 Collator::get_skip_externals_queue_size() { } void Collator::finalize_stats() { - double work_time = work_timer_.elapsed(); - double cpu_work_time = cpu_work_timer_.elapsed(); - LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + auto work_time = work_timer_.elapsed_both(); + LOG(WARNING) << "Collate query work time = " << work_time.real << "s, cpu time = " << work_time.cpu << "s"; if (block_candidate) { stats_.block_id = block_candidate->id; stats_.collated_data_hash = block_candidate->collated_file_hash; - stats_.actual_bytes = block_candidate->data.size(); - stats_.actual_collated_data_bytes = block_candidate->collated_data.size(); + stats_.actual_bytes = (td::uint32)block_candidate->data.size(); + stats_.actual_collated_data_bytes = (td::uint32)block_candidate->collated_data.size(); } else { stats_.block_id.id = new_id; } stats_.cc_seqno = validator_set_.not_null() ? validator_set_->get_catchain_seqno() : 0; stats_.collated_at = td::Clocks::system(); stats_.attempt = attempt_idx_; - stats_.is_validator = !(mode_ & CollateMode::from_collator_node); + stats_.is_validator = collator_node_id_.is_zero(); stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id() : collator_node_id_.pubkey_hash(); if (block_limit_status_) { - stats_.estimated_bytes = block_limit_status_->estimate_block_size(); - stats_.gas = block_limit_status_->gas_used; - stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; - stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_size_estimate; + stats_.estimated_bytes = (td::uint32)block_limit_status_->estimate_block_size(); + stats_.gas = (td::uint32)block_limit_status_->gas_used; + stats_.lt_delta = (td::uint32)(block_limit_status_->cur_lt - block_limit_status_->limits.start_lt); + stats_.estimated_collated_data_bytes = (td::uint32)block_limit_status_->collated_data_size_estimate; stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); @@ -6553,8 +6696,7 @@ void Collator::finalize_stats() { block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); } stats_.total_time = perf_timer_.elapsed(); - stats_.work_time = work_time; - stats_.cpu_work_time = cpu_work_time; + stats_.work_time.total = work_time; stats_.time_stats = (PSTRING() << perf_log_); if (is_masterchain() && shard_conf_) { shard_conf_->process_shard_hashes([&](const block::McShardHash& shard) { diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index de928fbc5..5ceb842c9 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -191,59 +191,34 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act .release(); } -void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode) { +void run_validate_query(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise) { BlockSeqno seqno = 0; - for (auto& p : prev) { + for (auto& p : params.prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } - bool is_fake = mode & ValidateMode::fake; static std::atomic idx; - td::actor::create_actor(PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() - << ":" << (seqno + 1) << "#" << idx.fetch_add(1), - shard, min_masterchain_block_id, std::move(prev), std::move(candidate), - std::move(validator_set), local_validator_id, std::move(manager), timeout, - std::move(promise), mode) + td::actor::create_actor( + PSTRING() << (params.is_fake ? "fakevalidate" : "validateblock") << params.shard.to_str() << ":" << (seqno + 1) + << "#" << idx.fetch_add(1), + std::move(candidate), std::move(params), std::move(manager), timeout, std::move(promise)) .release(); } -void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, - td::Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, - adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, - int attempt_idx) { +void run_collate_query(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise) { BlockSeqno seqno = 0; - for (auto& p : prev) { + for (auto& p : params.prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1) - << (attempt_idx ? "_" + td::to_string(attempt_idx) : ""), - shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set), - creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise), - collator_node_id, std::move(cancellation_token), mode, attempt_idx) - .release(); -} - -void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) { - BlockSeqno seqno = 0; - for (auto& p : prev) { - if (p.seqno() > seqno) { - seqno = p.seqno(); - } - } - td::actor::create_actor( - PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, - std::move(prev), td::Ref{}, Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, - std::move(manager), timeout, std::move(promise), adnl::AdnlNodeIdShort::zero(), td::CancellationToken{}, 0, 0) + td::actor::create_actor(PSTRING() << "collate" << params.shard.to_str() << ":" << (seqno + 1) + << (params.attempt_idx ? "_" + td::to_string(params.attempt_idx) : ""), + std::move(params), std::move(manager), timeout, std::move(cancellation_token), + std::move(promise)) .release(); } diff --git a/validator/impl/ihr-message.cpp b/validator/impl/ihr-message.cpp index 4327b5dd3..75cf08bc7 100644 --- a/validator/impl/ihr-message.cpp +++ b/validator/impl/ihr-message.cpp @@ -99,7 +99,7 @@ td::Result> IhrMessageQ::create_ihr_message(td::BufferSlice dat "block header in the Merkle proof of an IHR message does not belong to the declared source block"); } vm::AugmentedDictionary out_msg_dict{vm::load_cell_slice_ref(extra.out_msg_descr), 256, - block::tlb::aug_OutMsgDescr}; + block::tlb::aug_OutMsgDescrDefault}; Bits256 key{ihr_msg->get_hash().bits()}; auto descr = out_msg_dict.lookup(key); out_msg_dict.reset(); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index ed5bf148c..798caf8ae 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -1220,12 +1220,13 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St mc_state_ = Ref(std::move(mc_state)); CHECK(mc_state_.not_null()); - auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries); - if (rconfig.is_error()) { - fatal_error("cannot extract library list block configuration from masterchain state"); - return; - } - auto config = rconfig.move_as_ok(); + auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), mc_state_->get_block_id(), + block::ConfigInfo::needLibraries); + if (rconfig.is_error()) { + fatal_error("cannot extract library list block configuration from masterchain state"); + return; + } + auto config = rconfig.move_as_ok(); if (false) { std::ostringstream os; @@ -1564,25 +1565,24 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St return true; } - bool LiteQuery::make_ancestor_block_proof(Ref &proof, Ref state_root, - const BlockIdExt &old_blkid) { - vm::MerkleProofBuilder mpb{std::move(state_root)}; - auto rconfig = block::ConfigInfo::extract_config(mpb.root(), block::ConfigInfo::needPrevBlocks); - if (rconfig.is_error()) { - return fatal_error( - "cannot extract previous block configuration from masterchain state while constructing Merkle proof for "s + - old_blkid.to_str()); - } - if (!rconfig.move_as_ok()->check_old_mc_block_id(old_blkid, true)) { - return fatal_error("cannot check that "s + old_blkid.to_str() + - " is indeed a previous masterchain block while constructing Merkle proof"); - } - if (!mpb.extract_proof_to(proof)) { - return fatal_error( - "error while constructing Merkle proof for old masterchain block "s + old_blkid.to_str()); - } - return true; - } +bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref mc_state, const BlockIdExt& old_blkid) { + vm::MerkleProofBuilder mpb{mc_state->root_cell()}; + auto rconfig = + block::ConfigInfo::extract_config(mpb.root(), mc_state->get_block_id(), block::ConfigInfo::needPrevBlocks); + if (rconfig.is_error()) { + return fatal_error( + "cannot extract previous block configuration from masterchain state while constructing Merkle proof for "s + + old_blkid.to_str()); + } + if (!rconfig.move_as_ok()->check_old_mc_block_id(old_blkid, true)) { + return fatal_error("cannot check that "s + old_blkid.to_str() + + " is indeed a previous masterchain block while constructing Merkle proof"); + } + if (!mpb.extract_proof_to(proof)) { + return fatal_error("error while constructing Merkle proof for old masterchain block "s + old_blkid.to_str()); + } + return true; +} void LiteQuery::continue_getAccountState() { LOG(DEBUG) << "continue getAccountState() query"; @@ -1619,33 +1619,32 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St } } - void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { - LOG(DEBUG) << "completing getAccountState() query"; - Ref proof1, proof2; - if (!make_state_root_proof(proof1)) { - return; - } - vm::MerkleProofBuilder pb{state_->root_cell()}; - block::gen::ShardStateUnsplit::Record sstate; - if (!tlb::unpack_cell(pb.root(), sstate)) { - fatal_error("cannot unpack state header"); - return; - } - vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, - block::tlb::aug_ShardAccounts}; - auto acc_csr = accounts_dict.lookup(acc_addr_); - if (mode_ & 0x80000000) { - auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), 0xFFFF); - if (config.is_error()) { - fatal_error(config.move_as_error()); - return; - } - auto rconfig = config.move_as_ok(); - rconfig->set_block_id_ext(mc_state_->get_block_id()); - acc_state_promise_.set_value( - std::make_tuple(std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig))); - return; - } +void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { + LOG(INFO) << "completing getAccountState() query"; + Ref proof1, proof2; + if (!make_state_root_proof(proof1)) { + return; + } + vm::MerkleProofBuilder pb{state_->root_cell()}; + block::gen::ShardStateUnsplit::Record sstate; + if (!tlb::unpack_cell(pb.root(), sstate)) { + fatal_error("cannot unpack state header"); + return; + } + vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, block::tlb::aug_ShardAccounts}; + auto acc_csr = accounts_dict.lookup(acc_addr_); + if (mode_ & 0x80000000) { + auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), mc_state_->get_block_id(), 0xFFFF); + if (config.is_error()) { + fatal_error(config.move_as_error()); + return; + } + auto rconfig = config.move_as_ok(); + acc_state_promise_.set_value(std::make_tuple( + std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig) + )); + return; + } Ref acc_root; if (acc_csr.not_null()) { @@ -1819,7 +1818,7 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice LOG(DEBUG) << "creating VM with gas limit " << gas_limit; // **** INIT VM **** auto r_config = block::ConfigInfo::extract_config( - mc_state_->root_cell(), + mc_state_->root_cell(), mc_state_->get_block_id(), block::ConfigInfo::needLibraries | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks); if (r_config.is_error()) { fatal_error(r_config.move_as_error()); @@ -2242,7 +2241,7 @@ void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector< if (mode & block::ConfigInfo::needPrevBlocks) { mode |= block::ConfigInfo::needCapabilities; } - auto res = block::ConfigInfo::extract_config(mpb.root(), mode); + auto res = block::ConfigInfo::extract_config(mpb.root(), keyblk ? base_blk_id_ : mc_state_->get_block_id(), mode); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -2759,48 +2758,45 @@ void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector< request_block_data(blkid); } - static td::Result> get_in_msg_metadata( - const Ref &in_msg_descr_root, const Ref &trans_root) { - vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, - block::tlb::aug_InMsgDescr}; - block::gen::Transaction::Record transaction; - if (!block::tlb::unpack_cell(trans_root, transaction)) { - return td::Status::Error("invalid Transaction in block"); - } - Ref msg = transaction.r1.in_msg->prefetch_ref(); - if (msg.is_null()) { - return nullptr; - } - td::Bits256 in_msg_hash = msg->get_hash().bits(); - Ref in_msg = in_msg_descr.lookup(in_msg_hash); - if (in_msg.is_null()) { - return td::Status::Error( - PSTRING() << "no InMsg in InMsgDescr for message with hash " << in_msg_hash.to_hex()); - } - int tag = block::gen::t_InMsg.get_tag(*in_msg); - if (tag != block::gen::InMsg::msg_import_imm && tag != block::gen::InMsg::msg_import_fin && - tag != block::gen::InMsg::msg_import_deferred_fin) { - return nullptr; - } - Ref msg_env = in_msg->prefetch_ref(); - if (msg_env.is_null()) { - return td::Status::Error( - PSTRING() << "no MsgEnvelope in InMsg for message with hash " << in_msg_hash.to_hex()); - } - block::tlb::MsgEnvelope::Record_std env; - if (!block::tlb::unpack_cell(std::move(msg_env), env)) { - return td::Status::Error( - PSTRING() << "failed to unpack MsgEnvelope for message with hash " << in_msg_hash.to_hex()); - } - if (!env.metadata) { - return nullptr; - } - block::MsgMetadata &metadata = env.metadata.value(); - return create_tl_object( - 0, metadata.depth, - create_tl_object(metadata.initiator_wc, metadata.initiator_addr), - metadata.initiator_lt); - } +static td::Result> get_in_msg_metadata( + const Ref& in_msg_descr_root, const Ref& trans_root) { + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, + block::tlb::aug_InMsgDescrDefault}; + block::gen::Transaction::Record transaction; + if (!block::tlb::unpack_cell(trans_root, transaction)) { + return td::Status::Error("invalid Transaction in block"); + } + Ref msg = transaction.r1.in_msg->prefetch_ref(); + if (msg.is_null()) { + return nullptr; + } + td::Bits256 in_msg_hash = msg->get_hash().bits(); + Ref in_msg = in_msg_descr.lookup(in_msg_hash); + if (in_msg.is_null()) { + return td::Status::Error(PSTRING() << "no InMsg in InMsgDescr for message with hash " << in_msg_hash.to_hex()); + } + int tag = block::gen::t_InMsg.get_tag(*in_msg); + if (tag != block::gen::InMsg::msg_import_imm && tag != block::gen::InMsg::msg_import_fin && + tag != block::gen::InMsg::msg_import_deferred_fin) { + return nullptr; + } + Ref msg_env = in_msg->prefetch_ref(); + if (msg_env.is_null()) { + return td::Status::Error(PSTRING() << "no MsgEnvelope in InMsg for message with hash " << in_msg_hash.to_hex()); + } + block::tlb::MsgEnvelope::Record_std env; + if (!block::tlb::unpack_cell(std::move(msg_env), env)) { + return td::Status::Error(PSTRING() << "failed to unpack MsgEnvelope for message with hash " << in_msg_hash.to_hex()); + } + if (!env.metadata) { + return nullptr; + } + block::MsgMetadata& metadata = env.metadata.value(); + return create_tl_object( + 0, metadata.depth, + create_tl_object(metadata.initiator_wc, metadata.initiator_addr), + metadata.initiator_lt); +} void LiteQuery::finish_listBlockTransactions(int mode, int req_count) { LOG(INFO) @@ -2925,38 +2921,36 @@ void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector< request_block_data(blkid); } - static td::Status process_all_in_msg_metadata(const Ref &in_msg_descr_root, - const std::vector> &trans_roots) { - vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, - block::tlb::aug_InMsgDescr}; - for (const Ref &trans_root: trans_roots) { - block::gen::Transaction::Record transaction; - if (!block::tlb::unpack_cell(trans_root, transaction)) { - return td::Status::Error("invalid Transaction in block"); - } - Ref msg = transaction.r1.in_msg->prefetch_ref(); - if (msg.is_null()) { - continue; - } - td::Bits256 in_msg_hash = msg->get_hash().bits(); - Ref in_msg = in_msg_descr.lookup(in_msg_hash); - if (in_msg.is_null()) { - return td::Status::Error( - PSTRING() << "no InMsg in InMsgDescr for message with hash " << in_msg_hash.to_hex()); - } - int tag = block::gen::t_InMsg.get_tag(*in_msg); - if (tag == block::gen::InMsg::msg_import_imm || tag == block::gen::InMsg::msg_import_fin || - tag == block::gen::InMsg::msg_import_deferred_fin) { - Ref msg_env = in_msg->prefetch_ref(); - if (msg_env.is_null()) { - return td::Status::Error( - PSTRING() << "no MsgEnvelope in InMsg for message with hash " << in_msg_hash.to_hex()); - } - vm::load_cell_slice(msg_env); - } - } - return td::Status::OK(); - } +static td::Status process_all_in_msg_metadata(const Ref& in_msg_descr_root, + const std::vector>& trans_roots) { + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, + block::tlb::aug_InMsgDescrDefault}; + for (const Ref& trans_root : trans_roots) { + block::gen::Transaction::Record transaction; + if (!block::tlb::unpack_cell(trans_root, transaction)) { + return td::Status::Error("invalid Transaction in block"); + } + Ref msg = transaction.r1.in_msg->prefetch_ref(); + if (msg.is_null()) { + continue; + } + td::Bits256 in_msg_hash = msg->get_hash().bits(); + Ref in_msg = in_msg_descr.lookup(in_msg_hash); + if (in_msg.is_null()) { + return td::Status::Error(PSTRING() << "no InMsg in InMsgDescr for message with hash " << in_msg_hash.to_hex()); + } + int tag = block::gen::t_InMsg.get_tag(*in_msg); + if (tag == block::gen::InMsg::msg_import_imm || tag == block::gen::InMsg::msg_import_fin || + tag == block::gen::InMsg::msg_import_deferred_fin) { + Ref msg_env = in_msg->prefetch_ref(); + if (msg_env.is_null()) { + return td::Status::Error(PSTRING() << "no MsgEnvelope in InMsg for message with hash " << in_msg_hash.to_hex()); + } + vm::load_cell_slice(msg_env); + } + } + return td::Status::OK(); +} void LiteQuery::finish_listBlockTransactionsExt(int mode, int req_count) { LOG(INFO) @@ -3427,42 +3421,42 @@ void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector< return request_proof_link(cur) && request_mc_block_state(cur); } - bool LiteQuery::construct_proof_link_back_cont(ton::BlockIdExt cur, ton::BlockIdExt next) { - LOG(INFO) << "continue constructing a backward proof link from " << cur.to_str() << " to " << next.to_str(); - CHECK(mc_state_.not_null() && proof_link_.not_null() && mc_state_->get_block_id() == cur && - proof_link_->block_id() == cur); - try { - // virtualize proof link - auto vres1 = proof_link_->get_virtual_root(); - if (vres1.is_error()) { - return fatal_error(vres1.move_as_error()); - } - auto vroot = vres1.ok().root; - // adjust dest_proof and is_key of the last link of existing proof - if (!adjust_last_proof_link(cur, vroot)) { - return false; - } - // construct proof that `mc_state_` is the state of `cur` - Ref state_proof, proof; - if (!make_state_root_proof(proof, mc_state_->root_cell(), vroot, cur)) { - return fatal_error("cannot construct proof for state of masterchain block "s + cur.to_str()); - } - // construct proof that `next` is listed in OldMcBlocksInfo of `mc_state_` - if (!make_ancestor_block_proof(state_proof, mc_state_->root_cell(), next)) { - return fatal_error("cannot prove that "s + next.to_str() + - " is in the previous block set of the masterchain state of " + cur.to_str()); - } - // create a BlockProofLink for cur -> next (without dest_proof) - auto &link = chain_->new_link(cur, next, !next.seqno()); - link.proof = std::move(proof); - link.state_proof = std::move(state_proof); - // continue constructing proof chain from `next` - return construct_proof_chain(next); - } catch (vm::VmVirtError &) { - return fatal_error("virtualization error during construction of backward proof link from "s + cur.to_str() + - " to " + next.to_str()); - } - } +bool LiteQuery::construct_proof_link_back_cont(ton::BlockIdExt cur, ton::BlockIdExt next) { + LOG(INFO) << "continue constructing a backward proof link from " << cur.to_str() << " to " << next.to_str(); + CHECK(mc_state_.not_null() && proof_link_.not_null() && mc_state_->get_block_id() == cur && + proof_link_->block_id() == cur); + try { + // virtualize proof link + auto vres1 = proof_link_->get_virtual_root(); + if (vres1.is_error()) { + return fatal_error(vres1.move_as_error()); + } + auto vroot = vres1.ok().root; + // adjust dest_proof and is_key of the last link of existing proof + if (!adjust_last_proof_link(cur, vroot)) { + return false; + } + // construct proof that `mc_state_` is the state of `cur` + Ref state_proof, proof; + if (!make_state_root_proof(proof, mc_state_->root_cell(), vroot, cur)) { + return fatal_error("cannot construct proof for state of masterchain block "s + cur.to_str()); + } + // construct proof that `next` is listed in OldMcBlocksInfo of `mc_state_` + if (!make_ancestor_block_proof(state_proof, mc_state_, next)) { + return fatal_error("cannot prove that "s + next.to_str() + + " is in the previous block set of the masterchain state of " + cur.to_str()); + } + // create a BlockProofLink for cur -> next (without dest_proof) + auto& link = chain_->new_link(cur, next, !next.seqno()); + link.proof = std::move(proof); + link.state_proof = std::move(state_proof); + // continue constructing proof chain from `next` + return construct_proof_chain(next); + } catch (vm::VmVirtError&) { + return fatal_error("virtualization error during construction of backward proof link from "s + cur.to_str() + + " to " + next.to_str()); + } +} bool LiteQuery::finish_proof_chain(ton::BlockIdExt id) { CHECK(chain_); diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 5fb5ccff3..580fd0ce4 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -245,7 +245,7 @@ class LiteQuery : public td::actor::Actor { bool make_shard_info_proof(Ref& proof, Ref& info, ShardIdFull shard, bool exact = true); bool make_shard_info_proof(Ref& proof, Ref& info, AccountIdPrefixFull prefix); bool make_shard_info_proof(Ref& proof, BlockIdExt& blkid, AccountIdPrefixFull prefix); - bool make_ancestor_block_proof(Ref& proof, Ref state_root, const BlockIdExt& old_blkid); + bool make_ancestor_block_proof(Ref& proof, Ref mc_state, const BlockIdExt& old_blkid); }; } // namespace validator diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index e3330ea42..dc56997a1 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -334,7 +334,7 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo return; } td::actor::send_closure( - manager_, &ValidatorManager::wait_block_state_short, block, 0, entry->timeout, + manager_, &ValidatorManager::wait_block_state_short, block, 0, entry->timeout, false, [=, SelfId = actor_id(this), manager = manager_, timeout = entry->timeout, retry_after = td::Timestamp::in(0.1)](td::Result> R) mutable { if (R.is_error()) { diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 1f068efdb..1114bdc13 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -376,9 +376,9 @@ td::Status MasterchainStateQ::mc_init() { td::Status MasterchainStateQ::mc_reinit() { auto res = block::ConfigInfo::extract_config( - root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | - block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks | - block::ConfigInfo::needWorkchainInfo); + root_cell(), blkid, + block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needShardHashes | + block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needWorkchainInfo); cur_validators_.reset(); next_validators_.reset(); if (res.is_error()) { @@ -386,7 +386,6 @@ td::Status MasterchainStateQ::mc_reinit() { } config_ = res.move_as_ok(); CHECK(config_); - CHECK(config_->set_block_id_ext(get_block_id())); cur_validators_ = config_->get_cur_validator_set(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index ac7f400f3..a1ab5026f 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -59,33 +59,29 @@ std::string ErrorCtx::as_string() const { /** * Constructs a ValidateQuery object. * - * @param shard The shard of the block being validated. - * @param min_masterchain_block_id The minimum allowed masterchain block reference for the block. - * @param prev A vector of BlockIdExt representing the previous blocks. * @param candidate The BlockCandidate to be validated. - * @param validator_set A reference to the ValidatorSet. + * @param params Validation parameters * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the validation. * @param promise The Promise to return the ValidateCandidateResult to. - * @param mode +1 - fake mode */ -ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, Ref validator_set, - PublicKeyHash local_validator_id, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, unsigned mode) - : shard_(shard) +ValidateQuery::ValidateQuery(BlockCandidate candidate, ValidateParams params, + td::actor::ActorId manager, td::Timestamp timeout, + td::Promise promise) + : shard_(params.shard) , id_(candidate.id) - , min_mc_block_id(min_masterchain_block_id) - , prev_blocks(std::move(prev)) + , min_mc_block_id(params.min_masterchain_block_id) + , prev_blocks(std::move(params.prev)) , block_candidate(std::move(candidate)) - , validator_set_(std::move(validator_set)) - , local_validator_id_(local_validator_id) + , validator_set_(std::move(params.validator_set)) + , local_validator_id_(params.local_validator_id) , manager(manager) , timeout(timeout) , main_promise(std::move(promise)) - , is_fake_(mode & ValidateMode::fake) + , is_fake_(params.is_fake) , shard_pfx_(shard_.shard) , shard_pfx_len_(ton::shard_prefix_length(shard_)) + , optimistic_prev_block_(std::move(params.optimistic_prev_block)) , perf_timer_("validateblock", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validateblock", duration); }) { @@ -350,14 +346,31 @@ void ValidateQuery::start_up() { // return; } } + if (optimistic_prev_block_.not_null()) { + if (is_masterchain()) { + fatal_error("optimistic validation in masterchain is not supported"); + return; + } + if (prev_blocks.size() != 1) { + fatal_error("optimistic prev block is not null, which is not allowed after merge"); + return; + } + if (prev_blocks[0] != optimistic_prev_block_->block_id()) { + fatal_error("optimistic prev block is not null, but has invalid block id"); + return; + } + LOG(WARNING) << "Optimistic prev block id = " << optimistic_prev_block_->block_id().to_str(); + } // 2. learn latest masterchain state and block id LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; ++pending; td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { + [self = get_self(), token = perf_log_.start_action("get_top_masterchain_state_block")]( + td::Result, BlockIdExt>> res) mutable { LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_latest_mc_state, + std::move(res), std::move(token)); }); // 3. unpack block candidate (while necessary data is being loaded) if (!unpack_block_candidate()) { @@ -366,28 +379,25 @@ void ValidateQuery::start_up() { } // 4. load state(s) corresponding to previous block(s) (not full-collated-data or masterchain) prev_states.resize(prev_blocks.size()); - if (is_masterchain() || !full_collated_data_) { - for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 4.1. load state - LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) -> void { - LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); - }); + if (is_masterchain() || !full_collated_data_) { + if (optimistic_prev_block_.is_null()) { + load_prev_states(); + } else { + if (!process_optimistic_prev_block()) { + return; + } } } // 4. request masterchain handle and state referred to in the block if (!is_masterchain()) { ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::get_block_handle, mc_blkid_, true, - [self = get_self()](td::Result res) { - LOG(DEBUG) << "got answer to get_block_handle() query for masterchain block"; - td::actor::send_closure_later(std::move(self), &ValidateQuery::got_mc_handle, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_handle, mc_blkid_, true, + [self = get_self(), token = perf_log_.start_action("get_block_handle")](td::Result res) mutable { + LOG(DEBUG) << "got answer to get_block_handle() query for masterchain block"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::got_mc_handle, std::move(res), + std::move(token)); + }); } else { if (prev_blocks[0] != mc_blkid_) { soft_reject_query("cannot validate masterchain block "s + id_.to_str() + @@ -399,16 +409,103 @@ void ValidateQuery::start_up() { // 5. get storage stat cache ++pending; LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager"; - td::actor::send_closure_later( - manager, &ValidatorManager::get_storage_stat_cache, - [self = get_self()](td::Result(const td::Bits256&)>> res) { - LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; - td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_storage_stat_cache, std::move(res)); - }); + td::actor::send_closure_later(manager, &ValidatorManager::get_storage_stat_cache, + [self = get_self(), token = perf_log_.start_action("get_storage_stat_cache")]( + td::Result(const td::Bits256&)>> res) mutable { + LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_storage_stat_cache, + std::move(res), std::move(token)); + }); // ... CHECK(pending); } +/** + * Load previous states from DB + */ +void ValidateQuery::load_prev_states() { + for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { + // 4.1. load state + LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, false, + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i)]( + td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res), + std::move(token)); + }); + } +} + +/** + * Load previous state for optimistic prev block to apply Merkle update to it + */ +bool ValidateQuery::process_optimistic_prev_block() { + std::vector prev_prev; + BlockIdExt mc_blkid; + bool after_split; + auto S = block::unpack_block_prev_blk_try(optimistic_prev_block_->root_cell(), optimistic_prev_block_->block_id(), + prev_prev, mc_blkid, after_split); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); + } + // 4.1. load state + if (prev_prev.size() == 1) { + LOG(DEBUG) << "sending wait_block_state() query for " << prev_prev[0].to_str() << " to Manager (opt)"; + ++pending; + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, false, + [self = get_self(), + token = perf_log_.start_action("opt wait_block_state")](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query (opt)"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state_optimistic, + std::move(res), std::move(token)); + }); + } else { + CHECK(prev_prev.size() == 2); + LOG(DEBUG) << "sending wait_block_state_merge() query for " << prev_prev[0].to_str() << " and " + << prev_prev[1].to_str() << " to Manager (opt)"; + ++pending; + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], priority(), timeout, + [self = get_self(), + token = perf_log_.start_action("opt wait_block_state_merge")](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state_optimistic, + std::move(res), std::move(token)); + }); + } + return true; +} + +/** + * Callback function called after retrieving previous state for optimistic prev block + * + * @param res The retrieved state. + */ +void ValidateQuery::after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token) { + token.finish(res); + LOG(DEBUG) << "in ValidateQuery::after_get_shard_state_optimistic()"; + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + td::RealCpuTimer timer; + work_timer_.resume(); + auto state = res.move_as_ok(); + auto S = state.write().apply_block(optimistic_prev_block_->block_id(), optimistic_prev_block_); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("apply error: ")); + return; + } + work_timer_.pause(); + stats_.work_time.optimistic_apply = timer.elapsed_both(); + after_get_shard_state(0, std::move(state), {}); +} + /** * Unpacks and validates a block candidate. * @@ -698,10 +795,12 @@ bool ValidateQuery::extract_collated_data() { void ValidateQuery::request_latest_mc_state() { ++pending; td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { + [self = get_self(), token = perf_log_.start_action("get_top_masterchain_state_block")]( + td::Result, BlockIdExt>> res) mutable { LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_latest_mc_state, + std::move(res), std::move(token)); }); } @@ -710,7 +809,9 @@ void ValidateQuery::request_latest_mc_state() { * * @param res The result of the retrieval of the latest masterchain state. */ -void ValidateQuery::after_get_latest_mc_state(td::Result, BlockIdExt>> res) { +void ValidateQuery::after_get_latest_mc_state(td::Result, BlockIdExt>> res, + td::PerfLogAction token) { + token.finish(res); LOG(WARNING) << "in ValidateQuery::after_get_latest_mc_state()"; --pending; if (res.is_error()) { @@ -751,7 +852,8 @@ void ValidateQuery::after_get_latest_mc_state(td::Result> res) { +void ValidateQuery::after_get_mc_state(td::Result> res, td::PerfLogAction token) { + token.finish(res); CHECK(!is_masterchain()); LOG(WARNING) << "in ValidateQuery::after_get_mc_state() for " << mc_blkid_.to_str(); --pending; @@ -776,7 +878,8 @@ void ValidateQuery::after_get_mc_state(td::Result> res) { * * @param res The result of retrieving the masterchain block handle. */ -void ValidateQuery::got_mc_handle(td::Result res) { +void ValidateQuery::got_mc_handle(td::Result res, td::PerfLogAction token) { + token.finish(res); LOG(DEBUG) << "in ValidateQuery::got_mc_handle() for " << mc_blkid_.to_str(); if (res.is_error()) { fatal_error(res.move_as_error()); @@ -784,14 +887,16 @@ void ValidateQuery::got_mc_handle(td::Result res) { } auto mc_handle = res.move_as_ok(); td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, - [self = get_self(), id = id_, mc_handle](td::Result> res) { + manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, false, + [self = get_self(), id = id_, mc_handle, + token = perf_log_.start_action("mc wait_block_state")](td::Result> res) mutable { LOG(DEBUG) << "got answer to wait_block_state() query for masterchain block"; if (res.is_ok() && mc_handle->id().seqno() > 0 && !mc_handle->inited_proof()) { res = td::Status::Error(-666, "reference masterchain block "s + mc_handle->id().to_str() + " for block " + id.to_str() + " does not have a valid proof"); } - td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, std::move(res)); + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, std::move(res), + std::move(token)); }); } @@ -800,7 +905,9 @@ void ValidateQuery::got_mc_handle(td::Result res) { * * @param res The retrieved storage stat cache. */ -void ValidateQuery::after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res) { +void ValidateQuery::after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token) { + token.finish(res); --pending; if (res.is_error()) { LOG(INFO) << "after_get_storage_stat_cache : " << res.error(); @@ -821,7 +928,8 @@ void ValidateQuery::after_get_storage_stat_cache(td::Result> res) { +void ValidateQuery::after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token) { + token.finish(res); LOG(WARNING) << "in ValidateQuery::after_get_shard_state(" << idx << ")"; --pending; if (res.is_error()) { @@ -901,7 +1009,7 @@ bool ValidateQuery::try_unpack_mc_state() { return fatal_error(-666, "latest masterchain state does not have a root cell"); } auto res = block::ConfigInfo::extract_config( - mc_state_root_, + mc_state_root_, mc_blkid_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needStateExtraRoot | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks | @@ -912,7 +1020,6 @@ bool ValidateQuery::try_unpack_mc_state() { } config_ = res.move_as_ok(); CHECK(config_); - config_->set_block_id_ext(mc_blkid_); ihr_enabled_ = config_->ihr_enabled(); create_stats_enabled_ = config_->create_stats_enabled(); if (config_->has_capabilities() && (config_->get_capabilities() & ~supported_capabilities())) { @@ -943,6 +1050,7 @@ bool ValidateQuery::try_unpack_mc_state() { return reject_query(PSTRING() << "vertical seqno mismatch: new block has " << vert_seqno_ << " while the masterchain configuration expects " << config_->get_vert_seqno()); } + global_version_ = config_->get_global_version(); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); if (prev_key_block_exists_) { prev_key_block_seqno_ = prev_key_block_.seqno(); @@ -1074,11 +1182,13 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; action_phase_cfg_.disable_anycast = config_->get_global_version() >= 10; action_phase_cfg_.disable_ihr_flag = config_->get_global_version() >= 11; + action_phase_cfg_.global_version = config_->get_global_version(); } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; serialize_cfg_.disable_anycast = config_->get_global_version() >= 10; serialize_cfg_.store_storage_dict_hash = config_->get_global_version() >= 11; + serialize_cfg_.size_limits = size_limits; } { // fetch block_grams_created @@ -1406,17 +1516,17 @@ bool ValidateQuery::compute_next_state() { } } auto r_config_info = block::ConfigInfo::extract_config( - state_root_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | - block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | - block::ConfigInfo::needStateExtraRoot | block::ConfigInfo::needAccountsRoot | - block::ConfigInfo::needSpecialSmc | block::ConfigInfo::needCapabilities); + state_root_, id_, + block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | + block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needStateExtraRoot | + block::ConfigInfo::needAccountsRoot | block::ConfigInfo::needSpecialSmc | + block::ConfigInfo::needCapabilities); if (r_config_info.is_error()) { return reject_query("cannot extract configuration from new masterchain state "s + mc_blkid_.to_str() + " : " + r_config_info.error().to_string()); } new_config_ = r_config_info.move_as_ok(); CHECK(new_config_); - new_config_->set_block_id_ext(id_); } return true; } @@ -1599,7 +1709,8 @@ bool ValidateQuery::request_neighbor_queues() { return fatal_error("neighbor from masterchain is not the last mc block"); } ++pending; - send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, mc_state_->message_queue()); + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, mc_state_->message_queue(), + td::PerfLogAction{}); ++i; continue; } @@ -1612,18 +1723,26 @@ bool ValidateQuery::request_neighbor_queues() { return reject_query("cannot fetch shard state from collated data", state.move_as_error()); } ++pending; - send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, state.move_as_ok()->message_queue()); + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, state.move_as_ok()->message_queue(), + td::PerfLogAction{}); ++i; } } else { for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str(); ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, - std::move(res)); - }); + if (int prev_idx = prev_block_idx(descr.blk_); prev_idx >= 0) { + td::actor::send_closure(actor_id(this), &ValidateQuery::got_neighbor_out_queue, i, + prev_states.at(prev_idx)->message_queue(), td::PerfLogAction{}); + } else { + send_closure_later( + manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "wait_block_message_queue #" << i)]( + td::Result> res) mutable { + td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, std::move(res), + std::move(token)); + }); + } ++i; } } @@ -1637,7 +1756,8 @@ bool ValidateQuery::request_neighbor_queues() { * @param i The index of the neighbor. * @param res The obtained outbound queue. */ -void ValidateQuery::got_neighbor_out_queue(int i, td::Result> res) { +void ValidateQuery::got_neighbor_out_queue(int i, td::Result> res, td::PerfLogAction token) { + token.finish(res); --pending; if (res.is_error()) { fatal_error(res.move_as_error()); @@ -1764,13 +1884,15 @@ bool ValidateQuery::request_aux_mc_state(BlockSeqno seqno, Ref> res) { - LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); - td::actor::send_closure_later(std::move(self), - &ValidateQuery::after_get_aux_shard_state, blkid, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, false, + [self = get_self(), blkid, + token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.seqno())]( + td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_aux_shard_state, blkid, std::move(res), + std::move(token)); + }); state.clear(); return true; } @@ -1800,7 +1922,9 @@ Ref ValidateQuery::get_aux_mc_state(BlockSeqno seqno) const { * @param blkid The BlockIdExt of the shard state. * @param res The result of retrieving the shard state. */ -void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res) { +void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, + td::PerfLogAction token) { + token.finish(res); LOG(DEBUG) << "in ValidateQuery::after_get_aux_shard_state(" << blkid.to_str() << ")"; --pending; if (res.is_error()) { @@ -2196,11 +2320,12 @@ bool ValidateQuery::check_shard_layout() { } if (!new_top_shard_blocks.empty()) { ++pending; - td::actor::send_closure(manager, &ValidatorManager::wait_verify_shard_blocks, std::move(new_top_shard_blocks), - [SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ValidateQuery::verified_shard_blocks, - R.move_as_status()); - }); + td::actor::send_closure( + manager, &ValidatorManager::wait_verify_shard_blocks, std::move(new_top_shard_blocks), + [SelfId = actor_id(this), + token = perf_log_.start_action("wait_verify_shard_blocks")](td::Result R) mutable { + td::actor::send_closure(SelfId, &ValidateQuery::verified_shard_blocks, R.move_as_status(), std::move(token)); + }); } return check_mc_validator_info(is_key_block_ || (now_ / ccvc.mc_cc_lifetime > prev_now_ / ccvc.mc_cc_lifetime)); } @@ -2375,11 +2500,13 @@ bool ValidateQuery::prepare_out_msg_queue_size() { out_msg_queue_size_known_ = true; for (size_t i = 0; i < prev_blocks.size(); ++i) { ++pending; - send_closure_later(manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], - [self = get_self(), i](td::Result res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_out_queue_size, i, - std::move(res)); - }); + send_closure_later( + manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "get_out_msg_queue_size #" << i)]( + td::Result res) mutable { + td::actor::send_closure(std::move(self), &ValidateQuery::got_out_queue_size, i, std::move(res), + std::move(token)); + }); } return true; } @@ -2392,7 +2519,8 @@ bool ValidateQuery::prepare_out_msg_queue_size() { * @param i The index of the previous block (0 or 1). * @param res The result object containing the size of the queue. */ -void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { +void ValidateQuery::got_out_queue_size(size_t i, td::Result res, td::PerfLogAction token) { + token.finish(res); --pending; if (res.is_error()) { fatal_error( @@ -2412,7 +2540,8 @@ void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { * * @param S The status of the operation (OK on success). */ -void ValidateQuery::verified_shard_blocks(td::Status S) { +void ValidateQuery::verified_shard_blocks(td::Status S, td::PerfLogAction token) { + token.finish(S); --pending; if (S.is_error()) { fatal_error(S.move_as_error_prefix("failed to verify shard blocks: ")); @@ -2702,17 +2831,19 @@ bool ValidateQuery::unpack_block_data() { auto outmsg_cs = vm::load_cell_slice_ref(std::move(extra.out_msg_descr)); // run some hand-written checks from block::tlb:: // (automatic tests from block::gen:: have been already run for the entire block) - if (!block::tlb::t_InMsgDescr.validate_upto(10000000, *inmsg_cs)) { + t_InMsgDescr.aug.global_version = global_version_; + t_OutMsgDescr.aug.global_version = global_version_; + if (!t_InMsgDescr.validate_upto(10000000, *inmsg_cs)) { return reject_query("InMsgDescr of the new block failed to pass handwritten validity tests"); } - if (!block::tlb::t_OutMsgDescr.validate_upto(10000000, *outmsg_cs)) { + if (!t_OutMsgDescr.validate_upto(10000000, *outmsg_cs)) { return reject_query("OutMsgDescr of the new block failed to pass handwritten validity tests"); } if (!block::tlb::t_ShardAccountBlocks.validate_ref(10000000, extra.account_blocks)) { return reject_query("ShardAccountBlocks of the new block failed to pass handwritten validity tests"); } - in_msg_dict_ = std::make_unique(std::move(inmsg_cs), 256, block::tlb::aug_InMsgDescr); - out_msg_dict_ = std::make_unique(std::move(outmsg_cs), 256, block::tlb::aug_OutMsgDescr); + in_msg_dict_ = std::make_unique(std::move(inmsg_cs), 256, t_InMsgDescr.aug); + out_msg_dict_ = std::make_unique(std::move(outmsg_cs), 256, t_OutMsgDescr.aug); account_blocks_dict_ = std::make_unique( vm::load_cell_slice_ref(std::move(extra.account_blocks)), 256, block::tlb::aug_ShardAccountBlocks); LOG(DEBUG) << "validating InMsgDescr"; @@ -3771,7 +3902,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) block::tlb::MsgEnvelope::Record_std env; // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool // src:MsgAddressInt dest:MsgAddressInt - // value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + // value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams // created_lt:uint64 created_at:uint32 = CommonMsgInfo; block::gen::CommonMsgInfo::Record_int_msg_info info; ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix; @@ -4333,7 +4464,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms block::tlb::MsgEnvelope::Record_std env; // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool // src:MsgAddressInt dest:MsgAddressInt - // value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + // value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams // created_lt:uint64 created_at:uint32 = CommonMsgInfo; block::gen::CommonMsgInfo::Record_int_msg_info info; ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix; @@ -5369,12 +5500,36 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << new_acc->storage_used.cells << " cells)"; storage_stat_cache_update_.emplace_back(dict_root, new_acc->storage_used.cells); + stats_.storage_stat_cache.hit_cnt++; + stats_.storage_stat_cache.hit_cells += new_acc->storage_used.cells; + } else if (new_acc->storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { + stats_.storage_stat_cache.miss_cnt++; + stats_.storage_stat_cache.miss_cells += new_acc->storage_used.cells; + } else { + stats_.storage_stat_cache.small_cnt++; + stats_.storage_stat_cache.small_cells += new_acc->storage_used.cells; } } } return new_acc; } +/** + * Gets IHR fee of the internal message + * + * In earlier versions (before 12) the field extra_flags was ihr_fee. + * Since version 12 ihr_fee is always zero. + * + * @param info CommonMsgInfo of the internal message + * @param global_version global version from ConfigParam 8 + * + * @returns IHR fee + */ +static td::RefInt256 get_ihr_fee(const block::gen::CommonMsgInfo::Record_int_msg_info &info, int global_version) { + // Legacy: extra_flags was previously ihr_fee + return global_version >= 12 ? td::zero_refint() : block::tlb::t_Grams.as_integer(std::move(info.extra_flags)); +} + /** * Checks the validity of a single transaction for a given account. * Performs transaction execution. @@ -5462,7 +5617,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT CHECK(money_imported.validate_unpack(info.value)); ihr_delivered = (in_msg_tag == block::gen::InMsg::msg_import_ihr); if (!ihr_delivered) { - money_imported += block::tlb::t_Grams.as_integer(info.ihr_fee); + money_imported += get_ihr_fee(info, global_version_); } CHECK(money_imported.is_valid()); } @@ -5531,7 +5686,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // unpack exported message value (from this transaction) block::CurrencyCollection msg_export_value; CHECK(msg_export_value.unpack(info.value)); - msg_export_value += block::tlb::t_Grams.as_integer(info.ihr_fee); + msg_export_value += get_ihr_fee(info, global_version_); msg_export_value += msg_env.fwd_fee_remaining; CHECK(msg_export_value.is_valid()); money_exported += msg_export_value; @@ -5780,6 +5935,12 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // .... std::unique_ptr trs = std::make_unique(account, trans_type, lt, now_, in_msg_root); + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.trx_tvm += trs->time_tvm; + stats_.work_time.trx_storage_stat += trs->time_storage_stat; + stats_.work_time.trx_other += timer.elapsed_both() - trs->time_tvm - trs->time_storage_stat; + }; if (in_msg_root.not_null()) { if (!trs->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { // inbound external message was not accepted @@ -6143,9 +6304,12 @@ bool ValidateQuery::check_special_message(Ref in_msg_root, const block if (block::tlb::t_Grams.as_integer(info.fwd_fee)->sgn()) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero fwd_fee"); } - if (block::tlb::t_Grams.as_integer(info.ihr_fee)->sgn()) { + if (get_ihr_fee(info, global_version_)->sgn()) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero ihr_fee"); } + if (block::tlb::t_Grams.as_integer(info.extra_flags)->sgn()) { + return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero extra_flags"); + } block::CurrencyCollection value; if (!value.validate_unpack(info.value)) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has an invalid value"); @@ -7029,10 +7193,8 @@ bool ValidateQuery::try_validate() { return true; } work_timer_.resume(); - cpu_work_timer_.resume(); SCOPE_EXIT { work_timer_.pause(); - cpu_work_timer_.pause(); }; try { if (!stage_) { @@ -7152,13 +7314,14 @@ bool ValidateQuery::try_validate() { * @returns True. */ bool ValidateQuery::save_candidate() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ValidateQuery::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ValidateQuery::written_candidate); - } - }); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), token = perf_log_.start_action("set_block_candidate")](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ValidateQuery::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ValidateQuery::written_candidate, std::move(token)); + } + }); td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); @@ -7173,7 +7336,8 @@ bool ValidateQuery::save_candidate() { * Callback function called after saving block candidate. * Finishes validation. */ -void ValidateQuery::written_candidate() { +void ValidateQuery::written_candidate(td::PerfLogAction token) { + token.finish(td::Status::OK()); finish_query(); } @@ -7181,25 +7345,26 @@ void ValidateQuery::written_candidate() { * Sends validation work time to manager. */ void ValidateQuery::record_stats(bool valid, std::string error_message) { - ValidationStats stats; - stats.block_id = id_; - stats.collated_data_hash = block_candidate.collated_file_hash; - stats.validated_at = td::Clocks::system(); - stats.self = local_validator_id_; - stats.valid = valid; + stats_.block_id = id_; + stats_.collated_data_hash = block_candidate.collated_file_hash; + stats_.validated_at = td::Clocks::system(); + stats_.self = local_validator_id_; + stats_.valid = valid; if (valid) { - stats.comment = (PSTRING() << "OK ts=" << now_); + stats_.comment = (PSTRING() << "OK ts=" << now_); } else { - stats.comment = std::move(error_message); + stats_.comment = std::move(error_message); } - stats.actual_bytes = block_candidate.data.size(); - stats.actual_collated_data_bytes = block_candidate.collated_data.size(); - stats.total_time = perf_timer_.elapsed(); - stats.work_time = work_timer_.elapsed(); - stats.cpu_work_time = cpu_work_timer_.elapsed(); + stats_.actual_bytes = (td::uint32)block_candidate.data.size(); + stats_.actual_collated_data_bytes = (td::uint32)block_candidate.collated_data.size(); + stats_.total_time = perf_timer_.elapsed(); + stats_.work_time.total = work_timer_.elapsed_both(); + stats_.time_stats = (PSTRING() << perf_log_); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; - LOG(WARNING) << "Validate query work time = " << stats.work_time << "s, cpu time = " << stats.cpu_work_time << "s"; - td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats)); + LOG(WARNING) << "Validate query work time = " << stats_.work_time.total.real + << "s, cpu time = " << stats_.work_time.total.cpu << "s"; + LOG(WARNING) << perf_log_; + td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats_)); } } // namespace validator diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 95e81cdec..f338f975c 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -18,6 +18,9 @@ */ #pragma once + +#include "block-parse.h" +#include "fabric.h" #include "interfaces/validator-manager.h" #include "vm/cells.h" #include "vm/dict.h" @@ -112,15 +115,13 @@ class ValidateQuery : public td::actor::Actor { return SUPPORTED_VERSION; } static constexpr long long supported_capabilities() { - return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | - ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages | ton::capFullCollatedData; + return capCreateStatsEnabled | capBounceMsgBody | capReportVersion | capShortDequeue | capStoreOutMsgQueueSize | + capMsgMetadata | capDeferMessages | capFullCollatedData; } public: - ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode = 0); + ValidateQuery(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise); private: int verbosity{3 * 1}; @@ -153,6 +154,7 @@ class ValidateQuery : public td::actor::Actor { td::BitArray<64> shard_pfx_; int shard_pfx_len_; td::Bits256 created_by_; + Ref optimistic_prev_block_; Ref prev_state_root_; Ref state_root_; @@ -190,6 +192,7 @@ class ValidateQuery : public td::actor::Actor { ton::LogicalTime max_shard_lt_{0}; int global_id_{0}; + int global_version_{0}; ton::BlockSeqno vert_seqno_{~0U}; bool ihr_enabled_{false}; bool create_stats_enabled_{false}; @@ -222,6 +225,8 @@ class ValidateQuery : public td::actor::Actor { std::map block_create_count_; unsigned block_create_total_{0}; + block::tlb::InMsgDescr t_InMsgDescr{0}; + block::tlb::OutMsgDescr t_OutMsgDescr{0}; std::unique_ptr in_msg_dict_, out_msg_dict_, account_blocks_dict_; block::ValueFlow value_flow_; block::CurrencyCollection import_created_, transaction_fees_, total_burned_{0}, fees_burned_{0}; @@ -253,6 +258,7 @@ class ValidateQuery : public td::actor::Actor { bool have_unprocessed_account_dispatch_queue_ = false; td::PerfWarningTimer perf_timer_; + td::PerfLog perf_log_; static constexpr td::uint32 priority() { return 2; @@ -269,8 +275,12 @@ class ValidateQuery : public td::actor::Actor { void alarm() override; void start_up() override; + void load_prev_states(); + bool process_optimistic_prev_block(); + void after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token); + bool save_candidate(); - void written_candidate(); + void written_candidate(td::PerfLogAction token); bool fatal_error(td::Status error); bool fatal_error(int err_code, std::string err_msg); @@ -290,16 +300,25 @@ class ValidateQuery : public td::actor::Actor { bool is_masterchain() const { return shard_.is_masterchain(); } + int prev_block_idx(const BlockIdExt& id) const { + for (size_t i = 0; i < prev_blocks.size(); ++i) { + if (prev_blocks[i] == id) { + return (int)i; + } + } + return -1; + } td::actor::ActorId get_self() { return actor_id(this); } void request_latest_mc_state(); - void after_get_latest_mc_state(td::Result, BlockIdExt>> res); - void after_get_mc_state(td::Result> res); - void got_mc_handle(td::Result res); - void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res); - void after_get_shard_state(int idx, td::Result> res); + void after_get_latest_mc_state(td::Result, BlockIdExt>> res, td::PerfLogAction token); + void after_get_mc_state(td::Result> res, td::PerfLogAction token); + void got_mc_handle(td::Result res, td::PerfLogAction token); + void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token); + void after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token); bool process_mc_state(Ref mc_state); bool try_unpack_mc_state(); bool fetch_config_params(); @@ -319,12 +338,12 @@ class ValidateQuery : public td::actor::Actor { bool unpack_one_prev_state(block::ShardState& ss, BlockIdExt blkid, Ref prev_state_root); bool split_prev_state(block::ShardState& ss); bool request_neighbor_queues(); - void got_neighbor_out_queue(int i, td::Result> res); + void got_neighbor_out_queue(int i, td::Result> res, td::PerfLogAction token); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); Ref get_aux_mc_state(BlockSeqno seqno) const; - void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); + void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, td::PerfLogAction token); bool check_one_shard(const block::McShardHash& info, const block::McShardHash* sibling, const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc, bool& is_new); @@ -334,8 +353,8 @@ class ValidateQuery : public td::actor::Actor { bool check_mc_validator_info(bool update_mc_cc); bool check_utime_lt(); bool prepare_out_msg_queue_size(); - void got_out_queue_size(size_t i, td::Result res); - void verified_shard_blocks(td::Status S); + void got_out_queue_size(size_t i, td::Result res, td::PerfLogAction token); + void verified_shard_blocks(td::Status S, td::PerfLogAction token); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, ton::ShardIdFull owner, bool allow_cur = false); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto, bool allow_cur = false); @@ -413,8 +432,8 @@ class ValidateQuery : public td::actor::Actor { return true; } - td::Timer work_timer_{true}; - td::ThreadCpuTimer cpu_work_timer_{true}; + td::RealCpuTimer work_timer_{true}; + ValidationStats stats_; void record_stats(bool valid, std::string error_message = ""); }; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 30269b169..1b227aa5f 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -56,6 +56,17 @@ struct AsyncSerializerState { UnixTime last_written_block_ts; }; +struct StorageStatCacheStats { + td::uint64 small_cnt = 0, small_cells = 0; + td::uint64 hit_cnt = 0, hit_cells = 0; + td::uint64 miss_cnt = 0, miss_cells = 0; + + tl_object_ptr tl() const { + return create_tl_object(small_cnt, small_cells, hit_cnt, hit_cells, + miss_cnt, miss_cells); + } +}; + struct CollationStats { BlockIdExt block_id{workchainInvalid, 0, 0, RootHash::zero(), FileHash::zero()}; td::Status status = td::Status::OK(); @@ -70,7 +81,7 @@ struct CollationStats { td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; - double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + double total_time = 0.0; std::string time_stats; td::uint32 transactions = 0; @@ -105,6 +116,34 @@ struct CollationStats { double load_fraction_externals = -1.0; double load_fraction_new_msgs = -1.0; + struct WorkTimeStats { + td::RealCpuTimer::Time total; + td::RealCpuTimer::Time optimistic_apply; + td::RealCpuTimer::Time queue_cleanup; + td::RealCpuTimer::Time prelim_storage_stat; + td::RealCpuTimer::Time trx_tvm; + td::RealCpuTimer::Time trx_storage_stat; + td::RealCpuTimer::Time trx_other; + td::RealCpuTimer::Time final_storage_stat; + td::RealCpuTimer::Time create_block; + td::RealCpuTimer::Time create_collated_data; + td::RealCpuTimer::Time create_block_candidate; + + std::string to_str(bool is_cpu) const { + return PSTRING() << "total=" << total.get(is_cpu) << " optimistic_apply=" << optimistic_apply.get(is_cpu) + << " queue_cleanup=" << queue_cleanup.get(is_cpu) + << " prelim_storage_stat=" << prelim_storage_stat.get(is_cpu) + << " trx_tvm=" << trx_tvm.get(is_cpu) << " trx_storage_stat=" << trx_storage_stat.get(is_cpu) + << " trx_other=" << trx_other.get(is_cpu) + << " final_storage_stat=" << final_storage_stat.get(is_cpu) + << " create_block=" << create_block.get(is_cpu) + << " create_collated_data=" << create_collated_data.get(is_cpu) + << " create_block_candidate=" << create_block_candidate.get(is_cpu); + } + }; + WorkTimeStats work_time; + StorageStatCacheStats storage_stat_cache; + tl_object_ptr tl() const { std::vector> shards_obj; for (const BlockIdExt& block_id : shard_configuration) { @@ -121,13 +160,13 @@ struct CollationStats { std::move(neighbors_obj)); return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, - actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time, cpu_work_time, - time_stats, + actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time.total.real, + work_time.total.cpu, time_stats, work_time.to_str(false), work_time.to_str(true), create_tl_object( estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, load_fraction_queue_cleanup, load_fraction_dispatch, load_fraction_internals, load_fraction_externals, load_fraction_new_msgs, limits_log), - std::move(block_stats)); + std::move(block_stats), storage_stat_cache.tl()); } }; @@ -139,12 +178,30 @@ struct ValidationStats { bool valid = false; std::string comment; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; - double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + double total_time = 0.0; + std::string time_stats; + + struct WorkTimeStats { + td::RealCpuTimer::Time total; + td::RealCpuTimer::Time optimistic_apply; + td::RealCpuTimer::Time trx_tvm; + td::RealCpuTimer::Time trx_storage_stat; + td::RealCpuTimer::Time trx_other; + + std::string to_str(bool is_cpu) const { + return PSTRING() << "total=" << total.get(is_cpu) << " optimistic_apply=" << optimistic_apply.get(is_cpu) + << " trx_tvm=" << trx_tvm.get(is_cpu) << " trx_storage_stat=" << trx_storage_stat.get(is_cpu) + << " trx_other=" << trx_other.get(is_cpu); + } + }; + WorkTimeStats work_time; + StorageStatCacheStats storage_stat_cache; tl_object_ptr tl() const { return create_tl_object( create_tl_block_id(block_id), collated_data_hash, validated_at, self.bits256_value(), valid, comment, - actual_bytes, actual_collated_data_bytes, total_time, work_time, cpu_work_time); + actual_bytes, actual_collated_data_bytes, total_time, work_time.total.real, work_time.total.cpu, + time_stats, work_time.to_str(false), work_time.to_str(true), storage_stat_cache.tl()); } }; @@ -255,7 +312,6 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast, int mode) = 0; - virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 2b27dcccb..fea19b082 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -287,9 +287,12 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { } Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); - run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref{true}, - actor_id(this), td::Timestamp::in(10.0), std::move(P), adnl::AdnlNodeIdShort::zero(), - td::CancellationToken{}, 0); + run_collate_query(CollateParams{.shard = shard_id, + .min_masterchain_block_id = last_masterchain_block_id_, + .prev = prev, + .creator = created_by, + .validator_set = val_set}, + actor_id(this), td::Timestamp::in(10.0), {}, std::move(P)); } void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, @@ -312,8 +315,13 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, @@ -493,14 +501,15 @@ void ValidatorManagerImpl::dec_pending_new_blocks() { } void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, - actor_id(this), td::Timestamp::in(10.0), std::move(P)) + actor_id(this), td::Timestamp::in(10.0), + td::Promise>{}, std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -512,14 +521,14 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } - td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); @@ -573,7 +582,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), 0, timeout, true, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -592,7 +601,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), 0, timeout, true, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), 0, timeout, std::move(promise)); @@ -667,7 +676,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, 0, timeout, std::move(P)); + wait_block_state(handle, 0, timeout, true, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index d3c9b2e8e..f4222735a 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -185,9 +185,9 @@ class ValidatorManagerImpl : public ValidatorManager { std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, td::Promise>> promise) override { @@ -302,8 +302,6 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override { } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - } void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override { diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 3798803bb..717ab3aa5 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -55,7 +55,9 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { }); LOG(ERROR) << "running collate query"; - run_collate_hardfork(shard_id, block_id, prev, actor_id(this), td::Timestamp::in(10.0), std::move(P)); + run_collate_query( + CollateParams{.shard = shard_id, .min_masterchain_block_id = block_id, .prev = prev, .is_hardfork = true}, + actor_id(this), td::Timestamp::in(10.0), {}, std::move(P)); } void ValidatorManagerImpl::created_candidate(BlockCandidate candidate) { @@ -165,14 +167,15 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, - actor_id(this), td::Timestamp::in(10.0), std::move(P)) + actor_id(this), td::Timestamp::in(10.0), + td::Promise>{}, std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -184,14 +187,14 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } - td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); @@ -245,7 +248,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), 0, timeout, false, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -264,7 +267,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), 0, timeout, false, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), 0, timeout, std::move(promise)); @@ -339,7 +342,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, 0, timeout, std::move(P)); + wait_block_state(handle, 0, timeout, true, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 76d10ceb3..3ccff9365 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -185,7 +185,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } - void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) { + void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override { UNREACHABLE(); } void get_cell_db_reader(td::Promise> promise) override; @@ -201,9 +202,9 @@ class ValidatorManagerImpl : public ValidatorManager { void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override { UNREACHABLE(); } - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, td::Promise>> promise) override { @@ -246,7 +247,7 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_value(td::Unit()); } void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data, int mode) { + td::BufferSlice data, int mode) override { callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } @@ -352,8 +353,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_block_broadcast(BlockBroadcast broadcast, int mode) override { } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - } void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override { diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 873bce389..bef4828cc 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -356,9 +356,9 @@ void ValidatorManagerMasterchainStarter::got_init_block_handle(BlockHandle handl handle_ = std::move(handle); if (!handle_->received_state()) { - LOG(ERROR) << "db inconsistent: last state ( " << handle_->id() << " ) not received"; if (!read_only_) { - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, 1, td::Timestamp::in(600.0), + LOG(ERROR) << "db inconsistent: last state ( " << handle_->id() << " ) not received"; + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, 1, td::Timestamp::in(600.0), true, [SelfId = actor_id(this), handle = handle_](td::Result> R) { td::actor::send_closure( SelfId, &ValidatorManagerMasterchainStarter::got_init_block_handle, handle); @@ -370,6 +370,7 @@ void ValidatorManagerMasterchainStarter::got_init_block_handle(BlockHandle handl }, td::Timestamp::in(0.1)); } + return; } if (!handle_->is_applied()) { diff --git a/validator/manager.cpp b/validator/manager.cpp index bfd862398..3a7d20233 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -238,8 +238,7 @@ void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Pro void ValidatorManagerImpl::validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno) { for (auto &[_, collator_node] : collator_nodes_) { if (collator_node.can_collate_shard(block_id.shard_full())) { - td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, block_id.shard_full(), - std::vector{block_id}, cc_seqno); + td::actor::send_closure(collator_node.actor, &CollatorNode::new_shard_block_accepted, block_id, cc_seqno); } } } @@ -579,7 +578,7 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); + wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), true, std::move(P)); } if (validating_masterchain()) { td::MultiPromise mp; @@ -599,8 +598,8 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refshard())) { - td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, desc->shard(), - std::vector{desc->block_id()}, desc->catchain_seqno()); + td::actor::send_closure(collator_node.actor, &CollatorNode::new_shard_block_accepted, desc->block_id(), + desc->catchain_seqno()); } } } @@ -796,7 +795,7 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto it0 = block_state_cache_.find(handle->id()); if (it0 != block_state_cache_.end()) { it0->second.ttl_ = td::Timestamp::in(30.0); @@ -805,33 +804,43 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); + auto P1 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), true); + }); + auto P2 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), false); }); auto id = td::actor::create_actor("waitstate", handle, priority, opts_, last_masterchain_state_, - actor_id(this), td::Timestamp::at(timeout.at() + 10.0), std::move(P), - get_block_persistent_state_to_download(handle->id())) + actor_id(this), td::Timestamp::at(timeout.at() + 10.0), std::move(P1), + std::move(P2), get_block_persistent_state_to_download(handle->id())) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); } - it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); + if (wait_store) { + it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); + } else if (it->second.preliminary_done_) { + promise.set_result(it->second.preliminary_result_); + return; + } else { + it->second.waiting_preliminary_.emplace_back(timeout, priority, std::move(promise)); + } auto X = it->second.get_timeout(); td::actor::send_closure(it->second.actor_, &WaitBlockState::update_timeout, X.first, X.second); } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), priority, timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), priority, timeout, - std::move(promise)); + wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); } @@ -954,7 +963,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), priority, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), priority, timeout, false, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -973,7 +982,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), priority, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), priority, timeout, false, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), priority, timeout, std::move(promise)); @@ -1048,7 +1057,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, priority, timeout, std::move(P)); + wait_block_state(handle, priority, timeout, false, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, @@ -1314,37 +1323,51 @@ void ValidatorManagerImpl::get_block_by_seqno_from_db(AccountIdPrefixFull accoun td::actor::send_closure(db_, &Db::get_block_by_seqno, account, seqno, std::move(promise)); } -void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { - if (R.is_ok()) { - block_state_cache_[handle->id()] = {R.ok(), td::Timestamp::in(30.0)}; - } +void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R, + bool preliminary) { auto it = wait_state_.find(handle->id()); - if (it != wait_state_.end()) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout) { - for (auto &X : it->second.waiting_) { - X.promise.set_error(S.clone()); - } - } else if (it->second.waiting_.size() != 0) { - auto X = it->second.get_timeout(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); - }); - auto id = td::actor::create_actor("waitstate", handle, X.second, opts_, last_masterchain_state_, - actor_id(this), X.first, std::move(P), - get_block_persistent_state_to_download(handle->id())) - .release(); - it->second.actor_ = id; - return; - } - } else { - auto r = R.move_as_ok(); + if (it == wait_state_.end()) { + return; + } + if (R.is_ok()) { + auto r = R.move_as_ok(); + for (auto &X : it->second.waiting_preliminary_) { + X.promise.set_result(r); + } + it->second.preliminary_done_ = true; + it->second.preliminary_result_ = r; + it->second.waiting_preliminary_.clear(); + if (!preliminary) { + block_state_cache_[handle->id()] = {r, td::Timestamp::in(30.0)}; for (auto &X : it->second.waiting_) { X.promise.set_result(r); } + wait_state_.erase(it); + } + } else if (!preliminary) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout) { + for (auto &X : it->second.waiting_) { + X.promise.set_error(S.clone()); + } + for (auto &X : it->second.waiting_preliminary_) { + X.promise.set_error(S.clone()); + } + wait_state_.erase(it); + } else if (!it->second.waiting_.empty() || !it->second.waiting_preliminary_.empty()) { + auto X = it->second.get_timeout(); + auto P1 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), true); + }); + auto P2 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), false); + }); + auto id = td::actor::create_actor("waitstate", handle, X.second, opts_, last_masterchain_state_, + actor_id(this), X.first, std::move(P1), std::move(P2), + get_block_persistent_state_to_download(handle->id())) + .release(); + it->second.actor_ = id; } - wait_state_.erase(it); } } @@ -1849,11 +1872,6 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, int mo callback_->send_broadcast(std::move(broadcast), mode); } -void ValidatorManagerImpl::send_validator_telemetry(PublicKeyHash key, - tl_object_ptr telemetry) { - callback_->send_validator_telemetry(key, std::move(telemetry)); -} - void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) { @@ -1997,7 +2015,6 @@ void ValidatorManagerImpl::start_up() { } validator_manager_init(opts_, actor_id(this), db_.get(), std::move(P)); - init_session_stats(); check_waiters_at_ = td::Timestamp::in(1.0); alarm_timestamp().relax(check_waiters_at_); @@ -2048,7 +2065,6 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { if (opts_->nonfinal_ls_queries_enabled()) { candidates_buffer_ = td::actor::create_actor("candidates-buffer", actor_id(this)); } - init_validator_telemetry(); auto Q = td::PromiseCreator::lambda( [SelfId = actor_id(this)](td::Result>> R) { @@ -2352,7 +2368,6 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(serializer_, &AsyncStateSerializer::update_last_known_key_block_ts, last_key_block_handle_->unix_time()); } - init_validator_telemetry(); } update_shard_overlays(); @@ -2771,7 +2786,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group td::actor::ActorId ValidatorManagerImpl::get_collation_manager(adnl::AdnlNodeIdShort adnl_id) { auto &actor = collation_managers_[adnl_id]; if (actor.empty()) { - actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), rldp2_); + actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), adnl_, rldp2_); } return actor.get(); } @@ -2875,7 +2890,7 @@ void ValidatorManagerImpl::got_next_gc_masterchain_handle(BlockHandle handle) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::got_next_gc_masterchain_state, std::move(handle), td::Ref{R.move_as_ok()}); }); - wait_block_state(handle, 0, td::Timestamp::in(60.0), std::move(P)); + wait_block_state(handle, 0, td::Timestamp::in(60.0), true, std::move(P)); } void ValidatorManagerImpl::got_next_gc_masterchain_state(BlockHandle handle, td::Ref state) { @@ -3668,7 +3683,6 @@ void ValidatorManagerImpl::update_options(td::Ref opts) td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::update_options, opts); } opts_ = std::move(opts); - init_session_stats(); } void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { @@ -3887,86 +3901,19 @@ void ValidatorManagerImpl::CheckedExtMsgCounter::before_query() { } } -void ValidatorManagerImpl::init_validator_telemetry() { - if (last_masterchain_state_.is_null()) { - return; - } - td::Ref validator_set = last_masterchain_state_->get_total_validator_set(0); - if (validator_set.is_null()) { - validator_telemetry_.clear(); - return; - } - std::set processed; - for (auto &key : temp_keys_) { - if (const ValidatorDescr *desc = validator_set->get_validator(key.bits256_value())) { - processed.insert(key); - adnl::AdnlNodeIdShort adnl_id; - if (desc->addr.is_zero()) { - adnl_id = adnl::AdnlNodeIdShort{ValidatorFullId{desc->key}.compute_short_id()}; - } else { - adnl_id = adnl::AdnlNodeIdShort{desc->addr}; - } - auto &telemetry = validator_telemetry_[key]; - if (telemetry.empty()) { - telemetry = td::actor::create_actor("telemetry", key, adnl_id, - opts_->zero_block_id().file_hash, actor_id(this)); - } - } - } - for (auto it = validator_telemetry_.begin(); it != validator_telemetry_.end();) { - if (processed.contains(it->first)) { - ++it; - } else { - it = validator_telemetry_.erase(it); - } - } -} - -void ValidatorManagerImpl::init_session_stats() { - if (opts_->get_session_logs_file() == session_stats_filename_) { - return; - } - session_stats_filename_ = opts_->get_session_logs_file(); - if (session_stats_filename_.empty()) { - session_stats_enabled_ = false; - session_stats_fd_.close(); - return; - } - auto r_fd = td::FileFd::open(session_stats_filename_, - td::FileFd::Flags::Write | td::FileFd::Flags::Append | td::FileFd::Create); - if (r_fd.is_error()) { - LOG(ERROR) << "Failed to open session stats file for writing: " << r_fd.move_as_error(); - session_stats_filename_.clear(); - session_stats_enabled_ = false; - return; - } - session_stats_fd_ = r_fd.move_as_ok(); - session_stats_enabled_ = true; -} - template void ValidatorManagerImpl::write_session_stats(const T &obj) { - if (!session_stats_enabled_) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { return; } auto s = td::json_encode(td::ToJson(*obj.tl()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - s += '\n'; - td::Slice slice{s}; - while (!slice.empty()) { - auto R = session_stats_fd_.write(slice); - if (R.is_error()) { - LOG(WARNING) << "Failed to write to session stats: " << R.move_as_error(); - } - if (R.ok() == 0) { - LOG(WARNING) << "Failed to write to session stats"; - } - slice.remove_prefix(R.ok()); - } - auto S = session_stats_fd_.sync(); - if (S.is_error()) { - LOG(WARNING) << "Failed to write to session stats: " << S; - } + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); } void ValidatorManagerImpl::init_shard_block_verifier(adnl::AdnlNodeIdShort local_id) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 95ec37105..ba0a98d3b 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -36,9 +36,8 @@ #include "token-manager.h" #include "queue-size-counter.hpp" #include "storage-stat-cache.hpp" -#include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" -#include "collator-node.hpp" +#include "collator-node/collator-node.hpp" #include "shard-block-verifier.hpp" #include "shard-block-retainer.hpp" #include "td/utils/LRUCache.h" @@ -166,9 +165,17 @@ class ValidatorManagerImpl : public ValidatorManager { WaitList() = default; std::pair get_timeout() const { + return get_timeout_impl(waiting_); + } + void check_timers() { + check_timers_impl(waiting_); + } + + protected: + static std::pair get_timeout_impl(const std::vector> &waiting) { td::Timestamp t = td::Timestamp::now(); td::uint32 prio = 0; - for (auto &v : waiting_) { + for (auto &v : waiting) { if (v.timeout.at() > t.at()) { t = v.timeout; } @@ -178,10 +185,10 @@ class ValidatorManagerImpl : public ValidatorManager { } return {td::Timestamp::at(t.at() + 10.0), prio}; } - void check_timers() { + static void check_timers_impl(std::vector> &waiting) { td::uint32 j = 0; - auto f = waiting_.begin(); - auto t = waiting_.end(); + auto f = waiting.begin(); + auto t = waiting.end(); while (f < t) { if (f->timeout.is_in_past()) { f->promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); @@ -192,16 +199,26 @@ class ValidatorManagerImpl : public ValidatorManager { j++; } } - waiting_.resize(j); + waiting.resize(j); } }; template - struct WaitListCaching : public WaitList { - bool done_ = false; - ResType result_; - td::Timestamp remove_at_; + struct WaitListPreliminary : WaitList { + std::vector> waiting_preliminary_; + bool preliminary_done_ = false; + ResType preliminary_result_; + + std::pair get_timeout() const { + auto t1 = WaitList::get_timeout_impl(this->waiting_); + auto t2 = WaitList::get_timeout_impl(waiting_preliminary_); + return {std::max(t1.first, t2.first), std::max(t1.second, t2.second)}; + } + void check_timers() { + WaitList::check_timers_impl(this->waiting_); + WaitList::check_timers_impl(waiting_preliminary_); + } }; - std::map>> wait_state_; + std::map>> wait_state_; std::map>> wait_block_data_; struct CachedBlockState { @@ -366,7 +383,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.insert(key); - init_validator_telemetry(); promise.set_value(td::Unit()); } void del_permanent_key(PublicKeyHash key, td::Promise promise) override { @@ -375,7 +391,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void del_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.erase(key); - init_validator_telemetry(); promise.set_value(td::Unit()); } @@ -441,9 +456,9 @@ class ValidatorManagerImpl : public ValidatorManager { std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, td::Promise>> promise) override; @@ -541,7 +556,6 @@ class ValidatorManagerImpl : public ValidatorManager { void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override; - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override; void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override; @@ -586,7 +600,7 @@ class ValidatorManagerImpl : public ValidatorManager { void register_block_handle(BlockHandle handle); - void finished_wait_state(BlockHandle handle, td::Result> R); + void finished_wait_state(BlockHandle handle, td::Result> R, bool preliminary); void finished_wait_data(BlockHandle handle, td::Result> R); void start_up() override; @@ -827,10 +841,6 @@ class ValidatorManagerImpl : public ValidatorManager { void iterate_temp_block_handles(std::function f) override; - std::map> validator_telemetry_; - - void init_validator_telemetry(); - struct Collator { td::actor::ActorOwn actor; std::set shards; @@ -848,11 +858,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn storage_stat_cache_; - bool session_stats_enabled_ = false; - std::string session_stats_filename_; - td::FileFd session_stats_fd_; - - void init_session_stats(); template void write_session_stats(const T &obj); diff --git a/validator/queue-size-counter.cpp b/validator/queue-size-counter.cpp index 4fe55ae31..487bdefc8 100644 --- a/validator/queue-size-counter.cpp +++ b/validator/queue-size-counter.cpp @@ -112,7 +112,7 @@ void QueueSizeCounter::get_queue_size_ex(ton::BlockIdExt block_id, bool calc_who } BlockHandle handle = R.move_as_ok(); td::actor::send_closure( - manager, &ValidatorManager::wait_block_state, handle, 0, td::Timestamp::in(10.0), + manager, &ValidatorManager::wait_block_state, handle, 0, td::Timestamp::in(10.0), false, [SelfId, handle](td::Result> R) mutable { if (R.is_error()) { td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, handle->id(), @@ -159,7 +159,7 @@ void QueueSizeCounter::get_queue_size_cont(BlockHandle handle, td::Ref> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, state->get_block_id(), R.move_as_error()); @@ -213,7 +213,7 @@ void QueueSizeCounter::process_top_shard_blocks() { return; } td::actor::send_closure( - manager, &ValidatorManager::wait_block_state_short, R.ok()->id(), 0, td::Timestamp::in(10.0), + manager, &ValidatorManager::wait_block_state_short, R.ok()->id(), 0, td::Timestamp::in(10.0), false, [=](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "Failed to get masterchain state: " << R.move_as_error(); diff --git a/validator/shard-block-retainer.cpp b/validator/shard-block-retainer.cpp index 528f6bcf7..8ef4d8f21 100644 --- a/validator/shard-block-retainer.cpp +++ b/validator/shard-block-retainer.cpp @@ -109,7 +109,7 @@ void ShardBlockRetainer::new_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(30.0), + manager_, &ValidatorManager::wait_block_state_short, desc->block_id(), 0, td::Timestamp::in(30.0), true, [SelfId = actor_id(this), desc](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "Wait block state for " << desc->block_id().to_str() << " : " << R.move_as_error(); diff --git a/validator/shard-block-verifier.cpp b/validator/shard-block-verifier.cpp index ebd611144..0403ff22a 100644 --- a/validator/shard-block-verifier.cpp +++ b/validator/shard-block-verifier.cpp @@ -131,7 +131,7 @@ void ShardBlockVerifier::process_message(adnl::AdnlNodeIdShort src, td::BufferSl int ShardBlockVerifier::get_config_shard_idx(const ShardIdFull& shard_id) const { for (size_t i = 0; i < config_->shards.size(); i++) { if (shard_intersects(shard_id, config_->shards[i].shard_id)) { - return i; + return (int)i; } } return -1; diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 55fe9cdd8..a0e2e4eb2 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -161,7 +161,7 @@ void ShardClient::download_masterchain_state() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, masterchain_block_handle_, - shard_client_priority(), td::Timestamp::in(600), std::move(P)); + shard_client_priority(), td::Timestamp::in(600), true, std::move(P)); } void ShardClient::got_masterchain_block_state(td::Ref state) { @@ -207,7 +207,7 @@ void ShardClient::apply_all_shards() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, shard->top_block_id(), - shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), true, std::move(Q)); } } for (const auto &[wc, desc] : masterchain_state_->get_workchain_list()) { @@ -222,7 +222,7 @@ void ShardClient::apply_all_shards() { }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, BlockIdExt{wc, shardIdAll, 0, desc->zerostate_root_hash, desc->zerostate_file_hash}, - shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), true, std::move(Q)); } } } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 25394ba95..8151d1efe 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -23,9 +23,8 @@ #include "td/utils/overloaded.h" #include "common/delay.h" #include "ton/lite-tl.hpp" -#include "ton/ton-tl.hpp" #include "td/utils/Random.h" -#include "collator-node.hpp" +#include "collator-node/collator-node.hpp" namespace ton { @@ -43,9 +42,7 @@ void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo return; } td::uint32 round_id = source_info.priority.round; - if (round_id > last_known_round_id_) { - last_known_round_id_ = round_id; - } + update_round_id(round_id); if (!started_) { promise.set_error(td::Status::Error(ErrorCode::notready, "cannot collate block: group not started")); return; @@ -70,11 +67,38 @@ void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache), std::move(R)); }; + + if (optimistic_generation_ && prev_block_ids_.size() == 1 && optimistic_generation_->prev == prev_block_ids_[0] && + optimistic_generation_->round == round_id) { + if (optimistic_generation_->result) { + P.set_value(optimistic_generation_->result.value().clone()); + } else { + optimistic_generation_->promises.push_back( + [=, SelfId = actor_id(this), P = std::move(P), + cancellation_token = + cancellation_token_source_.get_cancellation_token()](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ValidatorGroup::generate_block_candidate_cont, source_info, std::move(P), + std::move(cancellation_token)); + } else { + P.set_value(R.move_as_ok()); + } + }); + } + return; + } + generate_block_candidate_cont(source_info, std::move(P), cancellation_token_source_.get_cancellation_token()); +} + +void ValidatorGroup::generate_block_candidate_cont(validatorsession::BlockSourceInfo source_info, + td::Promise promise, + td::CancellationToken cancellation_token) { + TRY_STATUS_PROMISE(promise, cancellation_token.check()); td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, - source_info.priority, validator_set_, max_answer_size, - cancellation_token_source_.get_cancellation_token(), std::move(P), config_.proto_version); + source_info.priority, validator_set_, max_answer_size, std::move(cancellation_token), + std::move(promise), config_.proto_version); } void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, @@ -93,6 +117,9 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) { send_block_candidate_broadcast(c.candidate.id, c.candidate.data.clone()); } + if (!c.self_collated) { + block_collator_node_id_[c.candidate.id] = adnl::AdnlNodeIdShort{c.collator_node_id}; + } cache->result = std::move(c); for (auto &p : cache->promises) { p.set_value(cache->result.value().clone()); @@ -102,22 +129,54 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo } void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, - td::Promise> promise) { + td::Promise> promise, + td::optional optimistic_prev_block) { if (destroying_) { promise.set_error(td::Status::Error("validator session finished")); return; } + bool is_optimistic = (bool)optimistic_prev_block; + if (is_optimistic && shard_.is_masterchain()) { + promise.set_error(td::Status::Error("no optimistic validation in masterchain")); + return; + } td::uint32 round_id = source_info.priority.round; - if (round_id > last_known_round_id_) { - last_known_round_id_ = round_id; + if (!is_optimistic) { + update_round_id(round_id); } if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error(ErrorCode::notready, "too old")); return; } + if (is_optimistic && round_id > last_known_round_id_ + 1) { + promise.set_error(td::Status::Error(ErrorCode::notready, "too new")); + return; + } + if (is_optimistic && shard_.is_masterchain()) { + promise.set_error(td::Status::Error("optimistic validation in masterchain is not supported")); + return; + } auto next_block_id = create_next_block_id(block.id.root_hash, block.id.file_hash); block.id = next_block_id; + auto prev = prev_block_ids_; + if (is_optimistic) { + if (round_id > last_known_round_id_) { + ++block.id.id.seqno; + } + optimistic_prev_block.value().id.id = block.id.id; + --optimistic_prev_block.value().id.id.seqno; + if (round_id == last_known_round_id_) { + if (prev_block_ids_ != std::vector{optimistic_prev_block.value().id}) { + promise.set_error(td::Status::Error("wrong prev block for optimistic validation")); + return; + } + optimistic_prev_block = {}; + is_optimistic = false; + } else { + prev = {optimistic_prev_block.value().id}; + } + } CacheKey cache_key = block_to_cache_key(block); auto it = approved_candidates_cache_.find(cache_key); @@ -126,44 +185,69 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), source_info, block = block.clone(), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { - LOG(ERROR) << "failed to validate candidate: " << S; - } - delay_action( - [SelfId, source_info, block = std::move(block), promise = std::move(promise)]() mutable { - td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, std::move(source_info), - std::move(block), std::move(promise)); - }, - td::Timestamp::in(0.1)); - } else { - auto v = R.move_as_ok(); - v.visit(td::overloaded( - [&](UnixTime ts) { - td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, block_to_cache_key(block), ts); - td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, block.pubkey.as_bits256(), - block.id, block.collated_file_hash); - if (need_send_candidate_broadcast(source_info, block.id.is_masterchain())) { - td::actor::send_closure(SelfId, &ValidatorGroup::send_block_candidate_broadcast, block.id, - block.data.clone()); - } - promise.set_value({ts, false}); - }, - [&](CandidateReject reject) { - promise.set_error( - td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); - })); - } - }); + auto it2 = block_collator_node_id_.find(block.id); + adnl::AdnlNodeIdShort collator_node_id = + it2 == block_collator_node_id_.end() ? adnl::AdnlNodeIdShort::zero() : it2->second; + + auto P = td::PromiseCreator::lambda( + [=, SelfId = actor_id(this), block = block.clone(), + optimistic_prev_block = is_optimistic ? optimistic_prev_block.value().clone() : td::optional{}, + promise = std::move(promise), + collation_manager = collation_manager_](td::Result R) mutable { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { + LOG(ERROR) << "failed to validate candidate: " << S; + } + delay_action( + [SelfId, source_info, block = std::move(block), promise = std::move(promise), + optimistic_prev_block = std::move(optimistic_prev_block)]() mutable { + td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, std::move(source_info), + std::move(block), std::move(promise), std::move(optimistic_prev_block)); + }, + td::Timestamp::in(0.1)); + } else { + auto v = R.move_as_ok(); + v.visit(td::overloaded( + [&](UnixTime ts) { + td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, block_to_cache_key(block), ts); + td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, + block.pubkey.as_bits256(), block.id, block.collated_file_hash); + if (need_send_candidate_broadcast(source_info, block.id.is_masterchain())) { + td::actor::send_closure(SelfId, &ValidatorGroup::send_block_candidate_broadcast, block.id, + block.data.clone()); + } + promise.set_value({ts, false}); + }, + [&](CandidateReject reject) { + if (!collator_node_id.is_zero()) { + td::actor::send_closure(collation_manager, &CollationManager::ban_collator, collator_node_id, + PSTRING() << "bad candidate " << block.id.to_str() << " : " << reject.reason); + } + promise.set_error( + td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); + })); + } + }); if (!started_) { P.set_error(td::Status::Error(ErrorCode::notready, "validator group not started")); return; } VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; - run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, local_id_, + td::Ref optimistic_prev_block_data; + if (is_optimistic) { + TRY_RESULT_PROMISE_PREFIX_ASSIGN( + P, optimistic_prev_block_data, + create_block(optimistic_prev_block.value().id, std::move(optimistic_prev_block.value().data)), + "failed to parse optimistic prev block: "); + } + run_validate_query(std::move(block), + ValidateParams{.shard = shard_, + .min_masterchain_block_id = min_masterchain_block_id_, + .prev = std::move(prev), + .validator_set = validator_set_, + .local_validator_id = local_id_, + .optimistic_prev_block = optimistic_prev_block_data}, manager_, td::Timestamp::in(15.0), std::move(P)); } @@ -179,9 +263,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so td::Promise promise) { stats.cc_seqno = validator_set_->get_catchain_seqno(); td::uint32 round_id = source_info.priority.round; - if (round_id >= last_known_round_id_) { - last_known_round_id_ = round_id + 1; - } + update_round_id(round_id + 1); auto sig_set = create_signature_set(std::move(signatures)); validator_set_->check_signatures(root_hash, file_hash, sig_set).ensure(); auto approve_sig_set = create_signature_set(std::move(approve_signatures)); @@ -193,7 +275,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so return; } auto next_block_id = create_next_block_id(root_hash, file_hash); - LOG(WARNING) << "Accepted block " << next_block_id; + LOG(WARNING) << "Accepted block " << next_block_id.to_str(); stats.block_id = next_block_id; td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(stats)); auto block = @@ -234,8 +316,11 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so std::move(approve_sig_set), send_broadcast_mode, std::move(promise)); prev_block_ids_ = std::vector{next_block_id}; cached_collated_block_ = nullptr; - approved_candidates_cache_.clear(); cancellation_token_source_.cancel(); + if (optimistic_generation_ && optimistic_generation_->round == last_known_round_id_ && + optimistic_generation_->prev != next_block_id) { + optimistic_generation_ = {}; + } } void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, @@ -262,9 +347,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref } void ValidatorGroup::skip_round(td::uint32 round_id) { - if (round_id >= last_known_round_id_) { - last_known_round_id_ = round_id + 1; - } + update_round_id(round_id + 1); } void ValidatorGroup::get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, @@ -275,6 +358,77 @@ void ValidatorGroup::get_approved_candidate(PublicKey source, RootHash root_hash std::move(promise)); } +void ValidatorGroup::generate_block_optimistic(validatorsession::BlockSourceInfo source_info, + td::BufferSlice prev_block, RootHash prev_root_hash, + FileHash prev_file_hash, td::Promise promise) { + if (destroying_) { + promise.set_error(td::Status::Error("validator session finished")); + return; + } + if (shard_.is_masterchain()) { + promise.set_error(td::Status::Error("no optimistic generation in masterchain")); + return; + } + if (last_known_round_id_ + 1 != source_info.priority.round) { + promise.set_error(td::Status::Error("too old round")); + return; + } + if (optimistic_generation_ && optimistic_generation_->round >= source_info.priority.round) { + promise.set_error(td::Status::Error("optimistic generation already in progress")); + return; + } + BlockIdExt block_id{create_next_block_id_simple(), prev_root_hash, prev_file_hash}; + optimistic_generation_ = std::make_unique(); + optimistic_generation_->round = source_info.priority.round; + optimistic_generation_->prev = BlockIdExt{create_next_block_id_simple(), prev_root_hash, prev_file_hash}; + optimistic_generation_->promises.push_back(std::move(promise)); + + td::Promise P = [=, SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_optimistic, source_info, std::move(R)); + }; + LOG(WARNING) << "Optimistically generating next block after " << block_id.to_str(); + td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; + td::actor::send_closure(collation_manager_, &CollationManager::collate_block_optimistic, shard_, min_masterchain_block_id_, + block_id, std::move(prev_block), Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, + source_info.priority, validator_set_, max_answer_size, + optimistic_generation_->cancellation_token_source.get_cancellation_token(), std::move(P), + config_.proto_version); +} + +void ValidatorGroup::generated_block_optimistic(validatorsession::BlockSourceInfo source_info, + td::Result R) { + if (!optimistic_generation_ || optimistic_generation_->round != source_info.priority.round) { + return; + } + if (R.is_error()) { + LOG(WARNING) << "Optimistic generation failed: " << R.move_as_error(); + for (auto &promise : optimistic_generation_->promises) { + promise.set_error(R.error().clone()); + } + optimistic_generation_ = {}; + return; + } + GeneratedCandidate c = R.move_as_ok(); + if (!c.self_collated) { + block_collator_node_id_[c.candidate.id] = adnl::AdnlNodeIdShort{c.collator_node_id}; + } + optimistic_generation_->result = std::move(c); + for (auto &promise : optimistic_generation_->promises) { + promise.set_result(optimistic_generation_->result.value().clone()); + } + optimistic_generation_->promises.clear(); +} + +void ValidatorGroup::update_round_id(td::uint32 round) { + if (last_known_round_id_ >= round) { + return; + } + last_known_round_id_ = round; + if (optimistic_generation_ && optimistic_generation_->round < round) { + optimistic_generation_ = {}; + } +} + BlockIdExt ValidatorGroup::create_next_block_id(RootHash root_hash, FileHash file_hash) const { return BlockIdExt{create_next_block_id_simple(), root_hash, file_hash}; } @@ -316,7 +470,7 @@ std::unique_ptr ValidatorGroup::ma sha256_bits256(collated_data.as_slice()), data.clone(), collated_data.clone()}; td::actor::send_closure(id_, &ValidatorGroup::validate_block_candidate, std::move(source_info), - std::move(candidate), std::move(P)); + std::move(candidate), std::move(P), td::optional{}); } void on_generate_slot(validatorsession::BlockSourceInfo source_info, td::Promise promise) override { @@ -352,6 +506,29 @@ std::unique_ptr ValidatorGroup::ma td::actor::send_closure(id_, &ValidatorGroup::get_approved_candidate, source, root_hash, file_hash, collated_data_file_hash, std::move(promise)); } + void generate_block_optimistic(validatorsession::BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, + td::Promise promise) override { + td::actor::send_closure(id_, &ValidatorGroup::generate_block_optimistic, source_info, std::move(prev_block), + prev_root_hash, prev_file_hash, std::move(promise)); + } + void on_optimistic_candidate(validatorsession::BlockSourceInfo source_info, + validatorsession::ValidatorSessionRootHash root_hash, td::BufferSlice data, + td::BufferSlice collated_data, PublicKey prev_source, + validatorsession::ValidatorSessionRootHash prev_root_hash, td::BufferSlice prev_data, + td::BufferSlice prev_collated_data) override { + BlockCandidate candidate{Ed25519_PublicKey{source_info.source.ed25519_value().raw()}, + BlockIdExt{0, 0, 0, root_hash, sha256_bits256(data.as_slice())}, + sha256_bits256(collated_data.as_slice()), data.clone(), collated_data.clone()}; + BlockCandidate prev_candidate{Ed25519_PublicKey{prev_source.ed25519_value().raw()}, + BlockIdExt{0, 0, 0, prev_root_hash, sha256_bits256(prev_data.as_slice())}, + sha256_bits256(prev_collated_data.as_slice()), prev_data.clone(), + prev_collated_data.clone()}; + + td::actor::send_closure( + id_, &ValidatorGroup::validate_block_candidate, std::move(source_info), std::move(candidate), + [](td::Result>) mutable {}, std::move(prev_candidate)); + } private: td::actor::ActorId id_; @@ -385,14 +562,17 @@ void ValidatorGroup::create_session() { } CHECK(found); + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_adnl_id_); + config_.catchain_opts.broadcast_speed_multiplier = opts_->get_catchain_broadcast_speed_multiplier(); if (!config_.new_catchain_ids) { session_ = validatorsession::ValidatorSession::create(session_id_, config_, local_id_, std::move(vec), - make_validator_session_callback(), keyring_, adnl_, rldp_, + make_validator_session_callback(), keyring_, adnl_, rldp2_, overlays_, db_root_, "-", allow_unsafe_self_blocks_resync_); } else { session_ = validatorsession::ValidatorSession::create( - session_id_, config_, local_id_, std::move(vec), make_validator_session_callback(), keyring_, adnl_, rldp_, + session_id_, config_, local_id_, std::move(vec), make_validator_session_callback(), keyring_, adnl_, rldp2_, overlays_, db_root_ + "/catchains/", PSTRING() << "." << shard_.workchain << "." << shard_.shard << "." << validator_set_->get_catchain_seqno() << ".", @@ -407,16 +587,12 @@ void ValidatorGroup::create_session() { if (started_) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::start); } - - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); - td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_adnl_id_); } void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id) { prev_block_ids_ = prev; min_masterchain_block_id_ = min_masterchain_block_id; cached_collated_block_ = nullptr; - approved_candidates_cache_.clear(); started_ = true; if (init_) { @@ -534,7 +710,7 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( BlockIdExt id{next_block_id, candidate->id_->block_id_->root_hash_, candidate->id_->block_id_->file_hash_}; candidate->id_->block_id_ = create_tl_lite_block_id(id); candidate->available_ = - available_block_candidates_.count({candidate->id_->creator_, id, candidate->id_->collated_data_hash_}); + available_block_candidates_.contains({candidate->id_->creator_, id, candidate->id_->collated_data_hash_}); } auto result = create_tl_object(); diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index a7456cf9f..3df456efa 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -37,8 +37,11 @@ class ValidatorManager; class ValidatorGroup : public td::actor::Actor { public: void generate_block_candidate(validatorsession::BlockSourceInfo source_info, td::Promise promise); + void generate_block_candidate_cont(validatorsession::BlockSourceInfo source_info, + td::Promise promise, td::CancellationToken cancellation_token); void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, - td::Promise> promise); + td::Promise> promise, + td::optional optimistic_prev_block); void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash, FileHash file_hash, std::vector signatures, std::vector approve_signatures, @@ -52,6 +55,10 @@ class ValidatorGroup : public td::actor::Actor { BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const; BlockId create_next_block_id_simple() const; + void generate_block_optimistic(validatorsession::BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, td::Promise promise); + void generated_block_optimistic(validatorsession::BlockSourceInfo source_info, td::Result R); + void start(std::vector prev, BlockIdExt min_masterchain_block_id); void create_session(); void destroy(); @@ -156,6 +163,8 @@ class ValidatorGroup : public td::actor::Actor { std::shared_ptr cached_collated_block_; td::CancellationTokenSource cancellation_token_source_; + void update_round_id(td::uint32 round); + void generated_block_candidate(validatorsession::BlockSourceInfo source_info, std::shared_ptr cache, td::Result R); @@ -179,8 +188,24 @@ class ValidatorGroup : public td::actor::Actor { } std::set sent_candidate_broadcasts_; + std::map block_collator_node_id_; void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data); + + struct OptimisticGeneration { + td::uint32 round = 0; + BlockIdExt prev; + td::optional result; + td::CancellationTokenSource cancellation_token_source; + std::vector> promises; + + ~OptimisticGeneration() { + for (auto& promise : promises) { + promise.set_error(td::Status::Error(ErrorCode::cancelled, "Cancelled")); + } + } + }; + std::unique_ptr optimistic_generation_; }; } // namespace validator diff --git a/validator/validator-telemetry.cpp b/validator/validator-telemetry.cpp index 403dd6f9f..b870e8b84 100644 --- a/validator/validator-telemetry.cpp +++ b/validator/validator-telemetry.cpp @@ -51,7 +51,7 @@ void ValidatorTelemetry::start_up() { cpu_cores_ = r_cpu_cores.move_as_ok(); } - LOG(DEBUG) << "Initializing validator telemetry, key = " << key_ << ", adnl_id = " << local_id_; + LOG(DEBUG) << "Initializing validator telemetry, adnl_id = " << local_id_; alarm_timestamp().relax(send_telemetry_at_ = td::Timestamp::in(td::Random::fast(30.0, 60.0))); } @@ -81,7 +81,7 @@ void ValidatorTelemetry::send_telemetry() { .cpu_threads_count; LOG(DEBUG) << "Sending validator telemetry for adnl id " << local_id_; - td::actor::send_closure(manager_, &ValidatorManager::send_validator_telemetry, key_, std::move(telemetry)); + callback_->send_telemetry(std::move(telemetry)); } } // namespace ton::validator diff --git a/validator/validator-telemetry.hpp b/validator/validator-telemetry.hpp index 73908bdd1..d83641dfe 100644 --- a/validator/validator-telemetry.hpp +++ b/validator/validator-telemetry.hpp @@ -33,23 +33,23 @@ namespace ton::validator { class ValidatorManager; class ValidatorTelemetry : public td::actor::Actor { -public: - ValidatorTelemetry(PublicKeyHash key, adnl::AdnlNodeIdShort local_id, td::Bits256 zero_state_file_hash, - td::actor::ActorId manager) - : key_(key) - , local_id_(local_id) - , zero_state_file_hash_(zero_state_file_hash) - , manager_(std::move(manager)) { + public: + class Callback { + public: + virtual ~Callback() = default; + virtual void send_telemetry(tl_object_ptr telemetry) = 0; + }; + + ValidatorTelemetry(adnl::AdnlNodeIdShort local_id, std::unique_ptr callback) + : local_id_(local_id), callback_(std::move(callback)) { } void start_up() override; void alarm() override; -private: - PublicKeyHash key_; + private: adnl::AdnlNodeIdShort local_id_; - td::Bits256 zero_state_file_hash_; - td::actor::ActorId manager_; + std::unique_ptr callback_; std::string node_version_; std::string os_version_; @@ -61,6 +61,5 @@ class ValidatorTelemetry : public td::actor::Actor { void send_telemetry(); static constexpr double PERIOD = 600.0; - static constexpr td::uint32 MAX_SIZE = 8192; }; -} // namespace ton::validator \ No newline at end of file +} // namespace ton::validator diff --git a/validator/validator.h b/validator/validator.h index aa10060e6..893561522 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -244,7 +244,6 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise>> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; - virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; }; virtual ~ValidatorManagerInterface() = default; @@ -329,9 +328,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) = 0; - virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) = 0; - virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) = 0; virtual void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks,