Update up-to-master

.github/workflows/build-universal.yaml

@@ -64,9 +64,7 @@ jobs:
         if: runner.os == 'macOS'
         run: |
           brew install ninja pkg-config automake coreutils libtool autoconf \
-            texinfo wget ccache llvm@16 boost jemalloc
-        env:
-          HOMEBREW_NO_AUTO_UPDATE: 1
+            texinfo wget ccache boost jemalloc cmake
 
       - name: Setup Python
         uses: actions/setup-python@v5

.gitmodules

@@ -13,10 +13,6 @@
 [submodule "third-party/cppkafka"]
 	path = third-party/cppkafka
 	url = https://github.com/mfontanini/cppkafka.git
-[submodule "third-party/pybind11"]
-	path = third-party/pybind11
-	url = https://github.com/pybind/pybind11.git
-	branch = stable
 [submodule "third-party/blst"]
 	path = third-party/blst
 	url = https://github.com/supranational/blst.git

Changelog.md

@@ -1,3 +1,15 @@
+## 2025.10 Update
+
+1. [TVM version v12](./doc/GlobalVersions.md): full bounces, new `BTOS` and `HASHBU` instuctions, limit on contract size in masterchain.
+2. Optimistic collation/validation: allow nodes to generate and check block candidates before previous block is fully signed (not fully activated yet).
+3. Introduced custom block compression algorithm.
+4. Overlay improvements: improved overlay discovery on shard configuration update, private externals in custom overlays.
+5. Various improvements: session stats, telemetry in fast-sync overlay, earlier block broadcasts, limiting ttl for values in DHT, fixing search by utime in native blockexplorer, faster downloading candidates in validator session, parallelization of storing to cell_db, avoiding touching packfiles on startup.
+
+Besides the work of the core team, this update is based on the efforts of the Tonstudio team: @hacker-volodya @Shvandre; and @mkiesel (avoiding touching packfiles on startup).
+
+
+
 ## 2025.07 Accelerator Update
 
 Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators)

adnl/adnl-ext-client.cpp

@@ -31,6 +31,14 @@ void AdnlExtClientImpl::alarm() {
     next_create_at_ = td::Timestamp::in(next_alarm_);
     alarm_timestamp() = next_create_at_;
 
+    if (!dst_host_.empty()) {
+      auto S = dst_addr_.init_host_port(dst_host_);
+      if (S.is_error()) {
+        LOG(INFO) << "failed to connect to " << dst_host_ << ": " << S;
+        return;
+      }
+      LOG(DEBUG) << "resolved " << dst_host_ << " -> " << dst_addr_;
+    }
     auto fd = td::SocketFd::open(dst_addr_);
     if (fd.is_error()) {
       LOG(INFO) << "failed to connect to " << dst_addr_ << ": " << fd.move_as_error();
@@ -166,6 +174,12 @@ td::actor::ActorOwn<AdnlExtClient> AdnlExtClient::create(AdnlNodeIdFull dst, td:
   return td::actor::create_actor<AdnlExtClientImpl>("extclient", std::move(dst), dst_addr, std::move(callback));
 }
 
+td::actor::ActorOwn<AdnlExtClient> AdnlExtClient::create(AdnlNodeIdFull dst, std::string dst_host,
+                                                         std::unique_ptr<AdnlExtClient::Callback> callback) {
+  return td::actor::create_actor<AdnlExtClientImpl>("extclient", std::move(dst), std::move(dst_host),
+                                                    std::move(callback));
+}
+
 td::actor::ActorOwn<AdnlExtClient> AdnlExtClient::create(AdnlNodeIdFull dst, PrivateKey local_id,
                                                          td::IPAddress dst_addr,
                                                          std::unique_ptr<AdnlExtClient::Callback> callback) {

adnl/adnl-ext-client.h

@@ -40,6 +40,8 @@ class AdnlExtClient : public td::actor::Actor {
                           td::Promise<td::BufferSlice> promise) = 0;
   static td::actor::ActorOwn<AdnlExtClient> create(AdnlNodeIdFull dst, td::IPAddress dst_addr,
                                                    std::unique_ptr<AdnlExtClient::Callback> callback);
+  static td::actor::ActorOwn<AdnlExtClient> create(AdnlNodeIdFull dst, std::string dst_host,
+                                                   std::unique_ptr<AdnlExtClient::Callback> callback);
   static td::actor::ActorOwn<AdnlExtClient> create(AdnlNodeIdFull dst, PrivateKey local_id, td::IPAddress dst_addr,
                                                    std::unique_ptr<AdnlExtClient::Callback> callback);
 };

adnl/adnl-ext-client.hpp

@@ -71,6 +71,9 @@ class AdnlExtClientImpl : public AdnlExtClient {
   AdnlExtClientImpl(AdnlNodeIdFull dst_id, td::IPAddress dst_addr, std::unique_ptr<Callback> callback)
       : dst_(std::move(dst_id)), dst_addr_(dst_addr), callback_(std::move(callback)) {
   }
+  AdnlExtClientImpl(AdnlNodeIdFull dst_id, std::string dst_host, std::unique_ptr<Callback> callback)
+      : dst_(std::move(dst_id)), dst_host_(std::move(dst_host)), callback_(std::move(callback)) {
+  }
   AdnlExtClientImpl(AdnlNodeIdFull dst_id, PrivateKey local_id, td::IPAddress dst_addr,
                     std::unique_ptr<Callback> callback)
       : dst_(std::move(dst_id)), local_id_(local_id), dst_addr_(dst_addr), callback_(std::move(callback)) {
@@ -138,6 +141,7 @@ class AdnlExtClientImpl : public AdnlExtClient {
   AdnlNodeIdFull dst_;
   PrivateKey local_id_;
   td::IPAddress dst_addr_;
+  std::string dst_host_;
 
   std::unique_ptr<Callback> callback_;
 

assembly/native/build-universal-static.sh

@@ -45,9 +45,10 @@ rm -rf .ninja* CMakeCache.txt CMakeFiles
 # Detect OS and set compiler
 # ----------------------
 if [[ "$OSTYPE" == "darwin"* ]]; then
-  echo "Detected macOS"
-  export CC="$(brew --prefix llvm@16)/bin/clang"
-  export CXX="$(brew --prefix llvm@16)/bin/clang++"
+  echo "CC=$(xcrun -find clang)"  >> $GITHUB_ENV
+  echo "CXX=$(xcrun -find clang++)" >> $GITHUB_ENV
+#  export CC="$(brew --prefix llvm@16)/bin/clang"
+#  export CXX="$(brew --prefix llvm@16)/bin/clang++"
   export OPENSSL_LIBS="$OPENSSL_PATH/lib/libcrypto.a"
 else
   echo "Detected Linux"

blockchain-explorer/blockchain-explorer-query.cpp

@@ -525,8 +525,8 @@ HttpQueryBlockSearch::HttpQueryBlockSearch(std::map<std::string, std::string> op
   }
   if (opts.count("utime") == 1) {
     try {
-      seqno_ = static_cast<td::uint32>(std::stoull(opts["utime"]));
-      mode_ = 1;
+      utime_ = static_cast<td::uint32>(std::stoull(opts["utime"]));
+      mode_ = 4;
     } catch (...) {
       error_ = td::Status::Error("cannot parse utime");
       return;
@@ -1429,10 +1429,10 @@ void HttpQueryStatus::finish_query() {
         A << "<td>" << static_cast<td::int32>(x->ts_.at_unix()) << "</td>";
       }
       A << "</tr>\n";
-      for (td::uint32 i = 0; i < results_.ips.size(); i++) {
+      for (td::uint32 i = 0; i < results_.addrs.size(); i++) {
         A << "<tr>";
-        if (results_.ips[i].is_valid()) {
-          A << "<td>" << results_.ips[i].get_ip_str() << ":" << results_.ips[i].get_port() << "</td>";
+        if (!results_.addrs[i].empty()) {
+          A << "<td>" << results_.addrs[i] << "</td>";
         } else {
           A << "<td>hidden</td>";
         }

blockchain-explorer/blockchain-explorer.cpp

@@ -187,7 +187,7 @@ class CoreActor : public CoreActorInterface {
   std::mutex queue_mutex_;
   std::mutex res_mutex_;
   std::map<td::int32, std::shared_ptr<RemoteNodeStatus>> results_;
-  std::vector<td::IPAddress> addrs_;
+  std::vector<std::string> addrs_;
   static CoreActor* instance_;
   td::actor::ActorId<CoreActor> self_id_;
 
@@ -220,7 +220,7 @@ class CoreActor : public CoreActorInterface {
   }
   void get_results(td::uint32 max, td::Promise<RemoteNodeStatusList> promise) override {
     RemoteNodeStatusList r;
-    r.ips = hide_ips_ ? std::vector<td::IPAddress>{addrs_.size()} : addrs_;
+    r.addrs = hide_ips_ ? std::vector<std::string>{addrs_.size()} : addrs_;
     auto it = results_.rbegin();
     while (it != results_.rend() && r.results.size() < max) {
       r.results.push_back(it->second);
@@ -445,14 +445,14 @@ class CoreActor : public CoreActorInterface {
       r_servers.ensure();
       servers = r_servers.move_as_ok();
       for (const auto& serv : servers) {
-        addrs_.push_back(serv.addr);
+        addrs_.push_back(serv.hostname);
       }
     } else {
       if (!remote_addr_.is_valid()) {
         LOG(FATAL) << "remote addr not set";
       }
-      addrs_.push_back(remote_addr_);
       servers.push_back(liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_});
+      addrs_.push_back(servers.back().hostname);
     }
     n_servers_ = servers.size();
     client_ = liteclient::ExtClient::create(std::move(servers), make_callback(), true);

blockchain-explorer/blockchain-explorer.hpp

@@ -59,7 +59,7 @@ class CoreActorInterface : public td::actor::Actor {
   };
 
   struct RemoteNodeStatusList {
-    std::vector<td::IPAddress> ips;
+    std::vector<std::string> addrs;
     std::vector<std::shared_ptr<RemoteNodeStatus>> results;
   };
   virtual ~CoreActorInterface() = default;

blockchain-indexer/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
+cmake_minimum_required(VERSION  3.16 FATAL_ERROR)
 
 
 if (NOT OPENSSL_FOUND)

blockchain-indexer/indexer.cpp

@@ -1224,7 +1224,7 @@ class IndexerWorker : public td::actor::Actor {
       */
 
         auto in_msg_dict = std::make_unique<vm::AugmentedDictionary>(vm::load_cell_slice_ref(extra.in_msg_descr), 256,
-                                                                     block::tlb::aug_InMsgDescr);
+                                                                     block::tlb::aug_InMsgDescrDefault);
 
         //          std::vector<json> in_msgs_json;
         //          while (!in_msg_dict->is_empty()) {
@@ -1985,10 +1985,6 @@ class Indexer : public td::actor::Actor {
         //        td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle));
       }
 
-      void send_validator_telemetry(ton::PublicKeyHash key,
-                                    ton::tl_object_ptr<ton::ton_api::validator_telemetry> telemetry) override {
-      }
-
       Callback(td::actor::ActorId<Indexer> id) : id_(id) {
       }
 
@@ -2268,10 +2264,6 @@ class IndexerSimple : public td::actor::Actor {
         //        td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle));
       }
 
-      void send_validator_telemetry(ton::PublicKeyHash key,
-                                    ton::tl_object_ptr<ton::ton_api::validator_telemetry> telemetry) override {
-      }
-
       Callback(td::actor::ActorId<IndexerSimple> id) : id_(id) {
       }
 

blockchain-indexer/json-utils.cpp

@@ -344,7 +344,7 @@ json parse_message(Ref<vm::Cell> message_any) {
         {"grams", block::tlb::t_Grams.as_integer(value_cc.grams)->to_dec_string()},
         {"extra", value_cc.other->have_refs() ? parse_extra_currency(value_cc.other->prefetch_ref()) : dummy}};
 
-    answer["ihr_fee"] = block::tlb::t_Grams.as_integer(msg.ihr_fee.write())->to_dec_string();
+    answer["extra_flags"] = block::tlb::t_Grams.as_integer(msg.extra_flags.write())->to_dec_string();
     answer["fwd_fee"] = block::tlb::t_Grams.as_integer(msg.fwd_fee.write())->to_dec_string();
     answer["created_lt"] = msg.created_lt;
     answer["created_at"] = msg.created_at;

common/global-version.h

@@ -19,6 +19,6 @@
 namespace ton {
 
 // See doc/GlobalVersions.md
-constexpr int SUPPORTED_VERSION = 11;
+constexpr int SUPPORTED_VERSION = 12;
 
 }

create-hardfork/create-hardfork.cpp

@@ -280,9 +280,6 @@ class HardforkCreator : public td::actor::Actor {
 
       void new_key_block(ton::validator::BlockHandle handle) override {
       }
-      void send_validator_telemetry(ton::PublicKeyHash key,
-                                    ton::tl_object_ptr<ton::ton_api::validator_telemetry> telemetry) override {
-      }
     };
 
     td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback,

crypto/CMakeLists.txt

@@ -14,6 +14,7 @@ set(TON_CRYPTO_CORE_SOURCE
   openssl/residue.cpp
   openssl/rand.cpp
   vm/boc.cpp
+  vm/boc-compression.cpp
   vm/large-boc-serializer.cpp
   tl/tlblib.cpp
 
@@ -109,6 +110,7 @@ set(TON_CRYPTO_SOURCE
   vm/arithops.h
   vm/atom.h
   vm/boc.h
+  vm/boc-compression.h
   vm/boc-writers.h
   vm/box.hpp
   vm/cellops.h

crypto/block/block-parse.cpp

@@ -665,7 +665,7 @@ bool CommonMsgInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const
              && t_MsgAddressInt.validate_skip(ops, cs, weak)       // src
              && t_MsgAddressInt.validate_skip(ops, cs, weak)       // dest
              && t_CurrencyCollection.validate_skip(ops, cs, weak)  // value
-             && t_Grams.validate_skip(ops, cs, weak)               // ihr_fee
+             && t_Grams.validate_skip(ops, cs, weak)               // extra_flags
              && t_Grams.validate_skip(ops, cs, weak)               // fwd_fee
              && cs.advance(64 + 32);                               // created_lt:uint64 created_at:uint32
     case ext_in_msg_info:
@@ -684,7 +684,7 @@ bool CommonMsgInfo::unpack(vm::CellSlice& cs, CommonMsgInfo::Record_int_msg_info
   return get_tag(cs) == int_msg_info && cs.advance(1) && cs.fetch_bool_to(data.ihr_disabled) &&
          cs.fetch_bool_to(data.bounce) && cs.fetch_bool_to(data.bounced) && t_MsgAddressInt.fetch_to(cs, data.src) &&
          t_MsgAddressInt.fetch_to(cs, data.dest) && t_CurrencyCollection.fetch_to(cs, data.value) &&
-         t_Grams.fetch_to(cs, data.ihr_fee) && t_Grams.fetch_to(cs, data.fwd_fee) &&
+         t_Grams.fetch_to(cs, data.extra_flags) && t_Grams.fetch_to(cs, data.fwd_fee) &&
          cs.fetch_uint_to(64, data.created_lt) && cs.fetch_uint_to(32, data.created_at);
 }
 
@@ -696,7 +696,7 @@ bool CommonMsgInfo::skip(vm::CellSlice& cs) const {
              && t_MsgAddressInt.skip(cs)       // src
              && t_MsgAddressInt.skip(cs)       // dest
              && t_CurrencyCollection.skip(cs)  // value
-             && t_Grams.skip(cs)               // ihr_fee
+             && t_Grams.skip(cs)               // extra_flags
              && t_Grams.skip(cs)               // fwd_fee
              && cs.advance(64 + 32);           // created_lt:uint64 created_at:uint32
     case ext_in_msg_info:
@@ -718,7 +718,7 @@ bool CommonMsgInfo::get_created_lt(vm::CellSlice& cs, unsigned long long& create
              && t_MsgAddressInt.skip(cs)             // src
              && t_MsgAddressInt.skip(cs)             // dest
              && t_CurrencyCollection.skip(cs)        // value
-             && t_Grams.skip(cs)                     // ihr_fee
+             && t_Grams.skip(cs)                     // extra_flags
              && t_Grams.skip(cs)                     // fwd_fee
              && cs.fetch_ulong_bool(64, created_lt)  // created_lt:uint64
              && cs.advance(32);                      // created_at:uint32
@@ -1843,26 +1843,18 @@ bool InMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const {
   return false;
 }
 
-bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const {
+static td::RefInt256 get_ihr_fee(const CommonMsgInfo::Record_int_msg_info &info, int global_version) {
+  // Legacy: extra_flags was previously ihr_fee
+  return global_version >= 12 ? td::zero_refint() : t_Grams.as_integer(std::move(info.extra_flags));
+}
+
+bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const {
   int tag = get_tag(cs);
   switch (tag) {
     case msg_import_ext:                   // inbound external message
       return t_ImportFees.null_value(cb);  // external messages have no value and no import fees
     case msg_import_ihr:                   // IHR-forwarded internal message to its final destination
-      if (cs.advance(3) && cs.size_refs() >= 3) {
-        auto msg_cs = load_cell_slice(cs.fetch_ref());
-        CommonMsgInfo::Record_int_msg_info msg_info;
-        td::RefInt256 ihr_fee;
-        vm::CellBuilder aux;
-        // sort of Prolog-style in C++
-        return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) &&
-               cs.fetch_ref().not_null() && (ihr_fee = t_Grams.as_integer_skip(cs)).not_null() &&
-               cs.fetch_ref().not_null() && !cmp(ihr_fee, t_Grams.as_integer(*msg_info.ihr_fee)) &&
-               cb.append_cellslice_bool(msg_info.ihr_fee)  // fees_collected := ihr_fee
-               && aux.append_cellslice_bool(msg_info.ihr_fee) && t_ExtraCurrencyCollection.null_value(aux) &&
-               t_CurrencyCollection.add_values(cb, aux.as_cellslice_ref().write(),
-                                               msg_info.value.write());  // value_imported := ihr_fee + value
-      }
+      // IHR is not implemented
       return false;
     case msg_import_imm:  // internal message re-imported from this very block
       if (cs.advance(3) && cs.size_refs() >= 2) {
@@ -1888,7 +1880,7 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const {
         return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) &&
                cb.append_cellslice_bool(in_msg.fwd_fee_remaining)  // fees_collected := fwd_fee_remaining
                && t_Grams.as_integer_skip_to(msg_info.value.write(), value_grams) &&
-               (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() &&
+               (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() &&
                t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) &&
                cb.append_cellslice_bool(
                    msg_info.value.write());  // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining
@@ -1911,7 +1903,7 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const {
         return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) &&
                t_Grams.store_integer_ref(cb, std::move(transit_fee))  // fees_collected := transit_fees
                && t_Grams.as_integer_skip_to(msg_info.value.write(), value_grams) &&
-               (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() &&
+               (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() &&
                t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) &&
                cb.append_cellslice_bool(
                    msg_info.value.write());  // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining
@@ -1941,8 +1933,8 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const {
 
 const InMsg t_InMsg;
 
-const Aug_InMsgDescr aug_InMsgDescr;
-const InMsgDescr t_InMsgDescr;
+const Aug_InMsgDescr aug_InMsgDescrDefault(ton::SUPPORTED_VERSION);
+const InMsgDescr t_InMsgDescrDefault(ton::SUPPORTED_VERSION);
 
 bool OutMsg::skip(vm::CellSlice& cs) const {
   switch (get_tag(cs)) {
@@ -2038,7 +2030,7 @@ bool OutMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const {
   return false;
 }
 
-bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const {
+bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const {
   auto tag = get_tag(cs);
   switch (tag) {
     case msg_export_ext:  // external outbound message carries no value
@@ -2071,7 +2063,7 @@ bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const {
         td::RefInt256 value_grams, ihr_fee, fwd_fee_remaining;
         return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) &&
                (value_grams = t_Grams.as_integer_skip(msg_info.value.write())).not_null() &&
-               (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() &&
+               (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() &&
                (fwd_fee_remaining = t_Grams.as_integer(out_msg.fwd_fee_remaining)).not_null() &&
                t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) &&
                cb.append_cellslice_bool(std::move(msg_info.value));
@@ -2112,8 +2104,8 @@ bool OutMsg::get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) c
 
 const OutMsg t_OutMsg;
 
-const Aug_OutMsgDescr aug_OutMsgDescr;
-const OutMsgDescr t_OutMsgDescr;
+const Aug_OutMsgDescr aug_OutMsgDescrDefault(ton::SUPPORTED_VERSION);
+const OutMsgDescr t_OutMsgDescrDefault(ton::SUPPORTED_VERSION);
 
 bool EnqueuedMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const {
   return cs.advance(64) && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak);