diff --git a/CMakeLists.txt b/CMakeLists.txt index 44eb54826..48c619cc4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,6 +69,12 @@ configure_file(${CMAKE_SOURCE_DIR}/cmake/CDTMacros.cmake.in ${CMAKE_BINARY_DIR}/ configure_file(${CMAKE_SOURCE_DIR}/cmake/CDTWasmToolchain.cmake.in ${CMAKE_BINARY_DIR}/cmake/CDTWasmToolchainPackage.cmake @ONLY) configure_file(${CMAKE_SOURCE_DIR}/cmake/cdt-config.cmake.in ${CMAKE_BINARY_DIR}/cmake/cdt-config.cmake.package @ONLY) +# Find zpp_bits header (from vcpkg) for contract compilation +find_path(ZPP_BITS_INCLUDE_DIR zpp_bits.h + HINTS ${CMAKE_PREFIX_PATH}/include ${VCPKG_INSTALLED_DIR}/x64-linux/include + REQUIRED) + + include(cmake/LibrariesExternalProject.cmake) include(cmake/InstallCDT.cmake) diff --git a/cmake/CDTMacros.cmake.in b/cmake/CDTMacros.cmake.in index ac0ee2414..8723825a2 100644 --- a/cmake/CDTMacros.cmake.in +++ b/cmake/CDTMacros.cmake.in @@ -34,6 +34,10 @@ macro(add_contract CONTRACT_NAME TARGET) set_cdt_include_directories(${TARGET}) endif() endmacro() +# Properties for protobuf support +define_property(TARGET PROPERTY PROTOBUF_FILES BRIEF_DOCS "protobuf files" FULL_DOCS "protobuf files used by the contract") +define_property(TARGET PROPERTY PROTOBUF_DIR BRIEF_DOCS "protobuf root directory" FULL_DOCS "protobuf root directory") + # Sets the Ricardian contract directory for a target # @param TARGET The target to set Ricardian directory for # @param DIR The directory containing Ricardian contract files @@ -198,3 +202,113 @@ macro(add_contract_native CONTRACT_NAME TARGET) endif() target_compile_options(${TARGET} PRIVATE -Wno-unknown-attributes) endmacro() + +# Generate C++ headers for protobuf files and tie to a given target +# +# target_add_protobuf(target +# INPUT_DIRECTORY input_dir +# OUTPUT_DIRECTORY output_dir +# FILES file [file1 ...]) +# +# Given a list of .proto files, this function generates the corresponding C++ headers +# using cdt-protoc-gen-zpp, and sets the generated files as sources for the specified target. +# +# If the input_dir is not specified, the default is ${CMAKE_CURRENT_SOURCE_DIR} and +# the files specified must be relative paths to the input_dir. +# +function(target_add_protobuf TARGET) + cmake_parse_arguments(ADD_PROTOBUF "" "INPUT_DIRECTORY;OUTPUT_DIRECTORY" "FILES" ${ARGN}) + + if (ADD_PROTOBUF_OUTPUT_DIRECTORY AND IS_ABSOLUTE ${ADD_PROTOBUF_OUTPUT_DIRECTORY}) + message(FATAL_ERROR "The OUTPUT_DIRECTORY for function target_add_protobuf must be a relative directory") + endif() + + if (ADD_PROTOBUF_OUTPUT_DIRECTORY) + set(OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/${ADD_PROTOBUF_OUTPUT_DIRECTORY}) + else() + set(OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}) + endif() + + foreach (protofile ${ADD_PROTOBUF_FILES}) + if(IS_ABSOLUTE ${protofile}) + message(FATAL_ERROR "The FILES parameters for function target_add_protobuf must be relative paths, illegal path `${protofile}`") + endif() + cmake_path(GET protofile EXTENSION LAST_ONLY proto_ext) + if (NOT proto_ext STREQUAL ".proto") + message(FATAL_ERROR "The illegal parameter `${protofile}` for function target_add_protobuf") + endif() + cmake_path(REPLACE_EXTENSION protofile LAST_ONLY ".pb.hpp" OUTPUT_VARIABLE hdr) + list(APPEND OUTPUT_HDRS ${OUTPUT_DIR}/${hdr}) + if (ADD_PROTOBUF_INPUT_DIRECTORY) + list(APPEND INPUT_FILES ${ADD_PROTOBUF_INPUT_DIRECTORY}/${protofile}) + else() + list(APPEND INPUT_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${protofile}) + endif() + endforeach() + + if (NOT ADD_PROTOBUF_INPUT_DIRECTORY) + set(ADD_PROTOBUF_INPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + find_program(PROTOC cdt-protoc + HINTS ${CMAKE_FIND_ROOT_PATH}/bin + REQUIRED) + + if (CMAKE_GENERATOR STREQUAL "Unix Makefiles" AND NOT EXISTS ${OUTPUT_DIR}) + file(MAKE_DIRECTORY ${OUTPUT_DIR}) + endif() + + add_custom_command( + COMMENT "Generating ${OUTPUT_HDRS} from ${INPUT_FILES}" + OUTPUT ${OUTPUT_HDRS} + COMMAND ${PROTOC} -I ${ADD_PROTOBUF_INPUT_DIRECTORY} -I ${CMAKE_FIND_ROOT_PATH}/include --plugin=protoc-gen-zpp=${CMAKE_FIND_ROOT_PATH}/bin/cdt-protoc-gen-zpp --zpp_out ${OUTPUT_DIR} ${ADD_PROTOBUF_FILES} + DEPENDS ${CMAKE_FIND_ROOT_PATH}/bin/cdt-protoc-gen-zpp ${INPUT_FILES} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) + + add_custom_target(${TARGET}.protos ALL + DEPENDS ${OUTPUT_HDRS} + ) + + get_target_property(type ${TARGET} TYPE) + if ("${type}" STREQUAL "INTERFACE_LIBRARY") + target_sources(${TARGET} INTERFACE ${OUTPUT_HDRS}) + target_include_directories(${TARGET} INTERFACE ${CMAKE_CURRENT_BINARY_DIR}) + else() + target_sources(${TARGET} PRIVATE ${OUTPUT_HDRS}) + target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) + add_dependencies(${TARGET} ${TARGET}.protos) + endif() + + set_target_properties(${TARGET} PROPERTIES + PROTOBUF_DIR "${ADD_PROTOBUF_INPUT_DIRECTORY}" + PROTOBUF_FILES "${ADD_PROTOBUF_FILES}") +endfunction() + +# Link a contract target to protobuf definitions for ABI generation +# +# contract_use_protobuf(contract_target protobuf_target) +# +function(contract_use_protobuf CONTRACT_TARGET PROTOBUF_TARGET) + get_target_property(PROTO_DIR ${PROTOBUF_TARGET} PROTOBUF_DIR) + get_target_property(PROTO_FILES ${PROTOBUF_TARGET} PROTOBUF_FILES) + set_target_properties(${CONTRACT_TARGET} PROPERTIES + PROTOBUF_DIR ${PROTO_DIR} + PROTOBUF_FILES "${PROTO_FILES}" + ) + # Propagate include directories from protobuf target for generated headers + get_target_property(PROTO_INCLUDES ${PROTOBUF_TARGET} INTERFACE_INCLUDE_DIRECTORIES) + if (PROTO_INCLUDES) + target_include_directories(${CONTRACT_TARGET} PUBLIC ${PROTO_INCLUDES}) + endif() + # Ensure protobuf headers are generated before the contract is compiled + if (TARGET ${PROTOBUF_TARGET}.protos) + add_dependencies(${CONTRACT_TARGET} ${PROTOBUF_TARGET}.protos) + endif() + # Pass protobuf info to cdt-codegen via compile options (picked up by cdt-cpp) + string(REPLACE ";" "$" PROTO_FILES_ESCAPED "${PROTO_FILES}") + target_compile_options(${CONTRACT_TARGET} PUBLIC + --protobuf-dir ${PROTO_DIR} + --protobuf-files "${PROTO_FILES_ESCAPED}" + ) +endfunction() diff --git a/cmake/InstallCDT.cmake b/cmake/InstallCDT.cmake index d764547c6..4a8fba808 100644 --- a/cmake/InstallCDT.cmake +++ b/cmake/InstallCDT.cmake @@ -57,6 +57,12 @@ cdt_tool_install_and_symlink(cdt-ld cdt-ld) cdt_tool_install_and_symlink(cdt-abidiff cdt-abidiff) cdt_tool_install_and_symlink(cdt-init cdt-init) cdt_tool_install_and_symlink(cdt-codegen cdt-codegen) +cdt_tool_install_and_symlink(cdt-protoc-gen-zpp cdt-protoc-gen-zpp) + +# Install cdt-protoc (protoc from vcpkg, copied during tools build) +add_custom_command( TARGET CDTTools POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/tools/bin/cdt-protoc ${CMAKE_BINARY_DIR}/bin/ ) +install(PROGRAMS ${CMAKE_BINARY_DIR}/tools/bin/cdt-protoc + DESTINATION ${CDT_INSTALL_PREFIX}/bin) # Sysio plugins (built by tools project) foreach(plugin sysio_attrs sysio_codegen) @@ -71,4 +77,17 @@ cdt_cmake_install_and_symlink(cdt-config.cmake cdt-config.cmake) cdt_cmake_install_and_symlink(CDTWasmToolchain.cmake CDTWasmToolchain.cmake) cdt_cmake_install_and_symlink(CDTMacros.cmake CDTMacros.cmake) +# Copy protobuf support files to main include dir for contract compilation +add_custom_command( TARGET CDTTools POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/include + COMMAND ${CMAKE_COMMAND} -E copy ${ZPP_BITS_INCLUDE_DIR}/zpp_bits.h ${CMAKE_BINARY_DIR}/include/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/include/zpp + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/tools/include/zpp/zpp_options.proto ${CMAKE_BINARY_DIR}/include/zpp/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/include/google/protobuf + COMMAND ${CMAKE_COMMAND} -E copy ${ZPP_BITS_INCLUDE_DIR}/google/protobuf/descriptor.proto ${CMAKE_BINARY_DIR}/include/google/protobuf/ ) +install(FILES ${CMAKE_BINARY_DIR}/tools/include/zpp/zpp_options.proto + DESTINATION ${CDT_INSTALL_PREFIX}/include/zpp) +install(FILES ${ZPP_BITS_INCLUDE_DIR}/google/protobuf/descriptor.proto + DESTINATION ${CDT_INSTALL_PREFIX}/include/google/protobuf) + cdt_libraries_install() diff --git a/docs/protocol-buffers.md b/docs/protocol-buffers.md new file mode 100644 index 000000000..268741953 --- /dev/null +++ b/docs/protocol-buffers.md @@ -0,0 +1,253 @@ +# Protocol Buffer Support + +Wire CDT supports [Protocol Buffers](https://protobuf.dev/) (protobuf) as an alternative serialization format for smart contract action data. This allows contracts to use protobuf's binary wire format instead of the default CDT datastream packing, providing: + +- **ID-based field encoding** for stable on-chain data formats +- **Language-neutral** message definitions with extensive library support +- **Backwards compatibility** for schema evolution (add/remove fields without breaking existing data) +- **Compact binary encoding** with fast serialization/deserialization + +## How It Works + +Instead of using the standard CDT serialization (which packs struct fields sequentially), protobuf actions use [zpp_bits](https://github.com/eyalz800/zpp_bits) — a header-only C++20 library that implements protobuf wire format without linking `libprotobuf`. This is critical because WASM contracts cannot use exceptions, and `zpp_bits` uses `std::errc` return codes for error handling. + +The workflow: + +1. Define messages in `.proto` files using proto3 syntax +2. `cdt-protoc-gen-zpp` (a custom protoc plugin) generates C++ structs with `zpp_bits` annotations +3. Wrap action parameters in `sysio::pb` to use protobuf serialization +4. The generated ABI includes a `protobuf_types` section with the FileDescriptorSet, enabling tools (clio, SDKs) to serialize/deserialize protobuf action data + +## Step 1: Define Proto Messages + +Create a `.proto` file with your message definitions: + +```protobuf +// mycontract.proto +syntax = "proto3"; +package mypackage; + +message TransferData { + string from = 1; + string to = 2; + uint64 amount = 3; + string memo = 4; +} + +message TransferResult { + uint64 balance = 1; +} +``` + +### Custom Field Options + +To use custom field options, add `import "zpp/zpp_options.proto";` at the top of your `.proto` file. + +- `[(zpp.zpp_type) = "sysio::name"]` — overrides the generated C++ type for a field + +## Step 2: Configure CMake + +```cmake +find_package(cdt) + +# Create an INTERFACE library for the proto definitions +add_library(my_protos INTERFACE) +target_add_protobuf(my_protos + OUTPUT_DIRECTORY mypackage + FILES mycontract.proto +) + +# Create the contract and link protobuf definitions +add_contract(mycontract mycontract mycontract.cpp) +contract_use_protobuf(mycontract my_protos) +``` + +The `target_add_protobuf()` function: +- Runs `cdt-protoc` with the `cdt-protoc-gen-zpp` plugin to generate `.pb.hpp` headers +- Adds the generated headers as sources to the target +- Sets up include directories so `#include ` works + +The `contract_use_protobuf()` function: +- Links the proto definitions to the contract for ABI generation +- Passes proto file information to `cdt-codegen` so the ABI includes `protobuf_types` + +## Step 3: Use in Contract Code + +### Single protobuf parameter (flattened) + +When an action has a single `sysio::pb` parameter, the ABI points the action type directly at the protobuf type — no wrapper struct is generated. This gives a clean, flat JSON interface: + +```cpp +#include +#include +#include + +namespace mypackage { + +class [[sysio::contract]] mycontract : public sysio::contract { +public: + using sysio::contract::contract; + + // Single pb param → action type is "protobuf::mypackage.TransferData" + [[sysio::action]] + sysio::pb transfer(const sysio::pb& data) { + sysio::check(static_cast(data.amount) > 0, "amount must be positive"); + // ... transfer logic ... + TransferResult result; + result.balance = zpp::bits::vuint64_t(new_balance); + return result; + } +}; + +} // namespace mypackage +``` + +JSON for pushing this action is flat — protobuf fields at the top level: + +```bash +clio push action mycontract transfer \ + '{"from":"alice","to":"bob","amount":1000,"memo":"payment"}' \ + -p alice@active +``` + +### Multiple protobuf parameters (wrapper struct) + +When an action has multiple parameters (protobuf or mixed), a wrapper struct is generated as usual, with each parameter as a named field: + +```cpp + // Multiple params → wrapper struct "settle" with fields "header" and "body" + [[sysio::action]] + void settle(const sysio::pb
& header, const sysio::pb& body) { + // ... + } +``` + +JSON includes the wrapper field names: + +```bash +clio push action mycontract settle \ + '{"header":{"version":1},"body":{"items":[...]}}' \ + -p alice@active +``` + +### Key points + +- Use `sysio::pb` to wrap protobuf message types in action parameters and return types +- The ABI generator detects `sysio::pb` and encodes the type as `protobuf::mypackage.TransferData` +- **Single `pb` parameter**: action type points directly at the protobuf type (flat JSON) +- **Multiple parameters**: a wrapper struct is generated (nested JSON) +- Protobuf integer types use `zpp::bits` varint wrappers (`vint32_t`, `vuint64_t`, etc.) +- Varint types don't implicitly convert — use `static_cast(field)` to access the underlying value + +## Generated ABI + +The generated `.abi` file uses version `sysio::abi/1.3` and includes a `protobuf_types` section containing the FileDescriptorSet in JSON format. Non-protobuf contracts continue to use `sysio::abi/1.2`. + +### Single parameter (flattened) + +The action type references the protobuf type directly. No wrapper struct is generated: + +```json +{ + "version": "sysio::abi/1.3", + "structs": [], + "actions": [ + { + "name": "transfer", + "type": "protobuf::mypackage.TransferData", + "ricardian_contract": "" + } + ], + "action_results": [ + { "name": "transfer", "result_type": "protobuf::mypackage.TransferResult" } + ], + "protobuf_types": { + "file": [ + { + "name": "mycontract.proto", + "package": "mypackage", + "messageType": [...] + } + ] + } +} +``` + +### Multiple parameters (wrapper struct) + +A wrapper struct is generated with one field per parameter: + +```json +{ + "version": "sysio::abi/1.3", + "structs": [ + { + "name": "settle", + "fields": [ + { "name": "header", "type": "protobuf::mypackage.Header" }, + { "name": "body", "type": "protobuf::mypackage.Body" } + ] + } + ], + "actions": [ + { + "name": "settle", + "type": "settle", + "ricardian_contract": "" + } + ], + "protobuf_types": { ... } +} +``` + +## Generated Code + +The `cdt-protoc-gen-zpp` plugin generates C++ structs with `zpp_bits` protobuf annotations. Each struct includes a `using serialize` declaration that tells `zpp_bits` to use protobuf wire format: + +```cpp +// Generated from mycontract.proto +namespace mypackage { +struct TransferData { + std::string from = {}; + std::string to = {}; + zpp::bits::vuint64_t amount = {}; + std::string memo = {}; + using serialize = zpp::bits::pb_members<4>; + bool operator == (const TransferData&) const = default; +}; +} // namespace mypackage +``` + +The `pb_members` declaration (where N is the number of fields) enables protobuf serialization. When fields have non-sequential proto field numbers, the generator uses `zpp::bits::protocol<...>` with explicit field number mappings instead. + +## Supported Proto3 Types + +| Proto3 Type | C++ Type | +|------------|----------| +| `int32` | `zpp::bits::vint32_t` | +| `int64` | `zpp::bits::vint64_t` | +| `uint32` | `zpp::bits::vuint32_t` | +| `uint64` | `zpp::bits::vuint64_t` | +| `sint32` | `zpp::bits::vsint32_t` | +| `sint64` | `zpp::bits::vsint64_t` | +| `fixed32` | `uint32_t` | +| `fixed64` | `uint64_t` | +| `sfixed32` | `int32_t` | +| `sfixed64` | `int64_t` | +| `float` | `float` | +| `double` | `double` | +| `bool` | `bool` | +| `string` | `std::string` | +| `bytes` | `std::vector` | +| `enum` | C++ `enum : int` | +| `message` | C++ `struct` | +| `repeated T` | `std::vector` | +| `map` | `std::map` | + +## Limitations + +- Only proto3 syntax is supported +- `oneof` fields are not supported +- `std::optional` fields (`[(zpp.zpp_optional) = true]`) are not supported — stock `zpp_bits` does not support optional fields in protobuf serialization mode +- Unpacked repeated fields are not supported +- WASM contracts have no exception support; serialization errors abort via `sysio::check()` diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index c7b55aa4d..3bd42e7bf 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -16,7 +16,7 @@ elseif(CMAKE_BUILD_TYPE STREQUAL "Debug") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") endif() -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_EXTENSIONS ON) add_subdirectory(libc) diff --git a/libraries/libc++/cdt-libcxx b/libraries/libc++/cdt-libcxx index 425fbc4eb..cd627a184 160000 --- a/libraries/libc++/cdt-libcxx +++ b/libraries/libc++/cdt-libcxx @@ -1 +1 @@ -Subproject commit 425fbc4eb4ebbfde6f5f20786816072dc96e5f95 +Subproject commit cd627a184e773e0954e6172ccc9552abbca1eaef diff --git a/libraries/sysiolib/core/sysio/pb.hpp b/libraries/sysiolib/core/sysio/pb.hpp new file mode 100644 index 000000000..23796d856 --- /dev/null +++ b/libraries/sysiolib/core/sysio/pb.hpp @@ -0,0 +1,107 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace sysio { + + /// Wrapper for protobuf message types used as action parameters. + /// When the ABI generator sees `sysio::pb`, it encodes the type as + /// `protobuf::fully.qualified.Name` instead of a normal struct, and the + /// action data is serialized/deserialized using protobuf wire format + /// (via zpp_bits) rather than the standard datastream packing. + template + struct pb : T { + using pb_message_type = T; + // Inherit the protobuf serialize protocol from T + using serialize = typename T::serialize; + serialize use(); + + pb() = default; + pb(const T& v) : T(v) {} + pb(T&& v) : T(std::move(v)) {} + pb(const pb&) = default; + pb(pb&&) = default; + + pb& operator=(const pb&) = default; + pb& operator=(pb&&) = default; + + template + pb(Args&&... args) : T(std::forward(args)...) {} + + bool operator==(const pb&) const = default; + }; + + template + const pb& to_pb(const T& v) { + return static_cast&>(v); + } + + template + T& from_pb(pb& v) { + return v; + } + + template + const T& from_pb(const pb& v) { + return v; + } + + template + T& from_pb(std::tuple>& v) { + return std::get<0>(v); + } + + template + const T& from_pb(const std::tuple>& v) { + return std::get<0>(v); + } + + // --- Protobuf serialization via zpp_bits --- + + /// Deserialize a pb from a datastream using protobuf wire format. + /// The stream must contain a varuint32 length prefix followed by the + /// protobuf-encoded message bytes. + template + datastream& operator>>(datastream& ds, pb& v) { + unsigned_int len; + ds >> len; + sysio::check(ds.remaining() >= len.value, "pb: not enough data for protobuf message"); + zpp::bits::in in(std::span(ds.pos(), len.value), zpp::bits::size_varint{}); + auto result = in(from_pb(v)); + sysio::check(result == std::errc{}, "protobuf deserialization failure"); + ds.skip(len.value); + return ds; + } + + /// Serialize a pb to a datastream using protobuf wire format. + /// Writes a varuint32 length prefix followed by the protobuf-encoded bytes. + template + datastream& operator<<(datastream& ds, const pb& v) { + std::vector result; + zpp::bits::out out(result, zpp::bits::size_varint{}); + auto status = out(from_pb(v)); + sysio::check(status == std::errc{}, "protobuf serialization failure"); + ds << unsigned_int(result.size()); + ds.write(result.data(), result.size()); + return ds; + } + + // --- zpp::bits varint JSON/binary serialization helpers --- + + template + void from_json(zpp::bits::varint& obj, auto& stream) { + T val; + from_json(val, stream); + obj = zpp::bits::varint(val); + } + + template + void to_json(zpp::bits::varint obj, auto& stream) { + to_json(static_cast(obj), stream); + } + +} // namespace sysio diff --git a/plugins/sysio/abigen.hpp b/plugins/sysio/abigen.hpp index c86cbd5c3..aca755ffc 100644 --- a/plugins/sysio/abigen.hpp +++ b/plugins/sysio/abigen.hpp @@ -124,7 +124,13 @@ namespace sysio { namespace cdt { validate_name( action_name.str(), [&](auto s) { CDT_ERROR("abigen_error", decl->getLocation(), s); } ); ret.name = action_name.str(); } - ret.type = decl->getNameAsString(); + // When a single pb parameter, point action type directly at protobuf type + if (is_single_pb_param(_decl)) { + auto param_type = _decl->parameters()[0]->getType().getNonReferenceType().getUnqualifiedType(); + ret.type = translate_type(param_type); + } else { + ret.type = decl->getNameAsString(); + } _abi.actions.insert(ret); // Handle action return types if (translate_type(decl->getReturnType()) != "void") { @@ -215,7 +221,22 @@ namespace sysio { namespace cdt { const auto res = _abi.structs.insert(ret); } + // Check if an action method has a single pb parameter, allowing the ABI + // to point directly at the protobuf type instead of generating a wrapper struct. + bool is_single_pb_param( const clang::CXXMethodDecl* decl ) { + if (decl->param_size() != 1) + return false; + auto param_type = decl->parameters()[0]->getType().getNonReferenceType().getUnqualifiedType(); + return is_template_specialization(param_type, {"pb"}); + } + void add_struct( const clang::CXXMethodDecl* decl ) { + // When an action has a single pb parameter, skip generating the wrapper + // struct — the action type points directly at the protobuf type. + if (is_single_pb_param(decl)) { + add_type(decl->parameters()[0]->getType().getNonReferenceType().getUnqualifiedType()); + return; + } abi_struct new_struct; new_struct.name = decl->getNameAsString(); for (auto param : decl->parameters() ) { @@ -515,6 +536,15 @@ namespace sysio { namespace cdt { else if (is_aliasing(type)) { add_typedef(type); } + else if (is_template_specialization(type, {"pb"})) { + // Protobuf types are not added as regular ABI structs. + // They are tracked via pb_types and embedded as protobuf_types in the ABI. + auto translated = translate_type(type); + if (translated.find("protobuf::") == 0) { + pb_types.insert(translated.substr(sizeof("protobuf::") - 1)); + } + return; + } else if (is_template_specialization(type, {"vector", "set", "deque", "list", "optional", "binary_extension", "ignore"})) { add_type(std::get(get_template_argument(type))); } @@ -848,6 +878,10 @@ namespace sysio { namespace cdt { for (auto& e : _abi.wasm_entries) { o["wasm_entries"].push_back(e); } + o["pb_types"] = ojson::array(); + for (auto& e : pb_types) { + o["pb_types"].push_back(e); + } return o; } @@ -857,6 +891,7 @@ namespace sysio { namespace cdt { std::set ctables; std::map rcs; std::set evaluated; + std::set pb_types; }; class sysio_abigen_visitor : public RecursiveASTVisitor, public generation_utils { diff --git a/plugins/sysio/gen.hpp b/plugins/sysio/gen.hpp index b285ad2f0..b44c0510f 100644 --- a/plugins/sysio/gen.hpp +++ b/plugins/sysio/gen.hpp @@ -685,6 +685,32 @@ struct generation_utils { if(is_explicit_nested(type)){ return translate_explicit_nested_type(type.getNonReferenceType()); } + else if ( is_template_specialization( type, {"pb"} ) ) { + // sysio::pb -> "protobuf::package.MessageType" + auto pt = llvm::dyn_cast(type.getTypePtr()); + auto tst = llvm::dyn_cast(pt ? pt->desugar().getTypePtr() : type.getTypePtr()); + if (tst && tst->template_arguments().size() > 0) { + auto arg = tst->template_arguments()[0]; + if (arg.getKind() == clang::TemplateArgument::ArgKind::Type) { + auto ctsd = arg.getAsType()->getAsCXXRecordDecl(); + if (ctsd) { + std::string message_type = ctsd->getQualifiedNameAsString(); + // Replace C++ :: with protobuf . + std::string result; + for (size_t i = 0; i < message_type.size(); ++i) { + if (i + 1 < message_type.size() && message_type[i] == ':' && message_type[i+1] == ':') { + result += '.'; + ++i; + } else { + result += message_type[i]; + } + } + return "protobuf::" + result; + } + } + } + return "bytes"; + } else if ( is_template_specialization( type, {"ignore"} ) ) return get_template_argument_as_string( type ); else if ( is_template_specialization( type, {"binary_extension"} ) ) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2d9209734..1ec0b7076 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -29,6 +29,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/unit/version_tests.sh ${CMAKE_BINARY_ add_test(NAME version_tests COMMAND ${CMAKE_BINARY_DIR}/tests/unit/version_tests.sh "${VERSION_FULL}" WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST version_tests PROPERTY LABELS unit_tests) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/unit/abi_version_tests.sh ${CMAKE_BINARY_DIR}/tests/unit/abi_version_tests.sh COPYONLY) +add_test(NAME abi_version_tests COMMAND ${CMAKE_BINARY_DIR}/tests/unit/abi_version_tests.sh "${CMAKE_BINARY_DIR}" WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST abi_version_tests PROPERTY LABELS unit_tests) + if (sysio_FOUND) add_test(integration_tests ${CMAKE_BINARY_DIR}/tests/integration/integration_tests) set_property(TEST integration_tests PROPERTY LABELS integration_tests) diff --git a/tests/unit/abi_version_tests.sh b/tests/unit/abi_version_tests.sh new file mode 100755 index 000000000..25231dd7c --- /dev/null +++ b/tests/unit/abi_version_tests.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Test ABI version and protobuf_types generation +# Usage: abi_version_tests.sh +set -euo pipefail + +BUILD_DIR="$1" +CONTRACTS_DIR="${BUILD_DIR}/tests/unit/test_contracts" +PASS=0 +FAIL=0 + +check() { + local desc="$1" file="$2" pattern="$3" + if grep -q "$pattern" "$file"; then + echo " PASS: $desc" + PASS=$((PASS + 1)) + else + echo " FAIL: $desc" + echo " expected pattern: $pattern" + echo " in file: $file" + FAIL=$((FAIL + 1)) + fi +} + +check_absent() { + local desc="$1" file="$2" pattern="$3" + if grep -q "$pattern" "$file" 2>/dev/null; then + echo " FAIL: $desc" + echo " pattern should NOT be present: $pattern" + echo " in file: $file" + FAIL=$((FAIL + 1)) + else + echo " PASS: $desc" + PASS=$((PASS + 1)) + fi +} + +echo "=== ABI Version Tests ===" + +# Non-protobuf contract should be ABI 1.2 +echo "-- simple_tests (non-protobuf) --" +check "version is 1.2" \ + "${CONTRACTS_DIR}/simple_tests.abi" \ + '"version": "sysio::abi/1.2"' +check_absent "no protobuf_types section" \ + "${CONTRACTS_DIR}/simple_tests.abi" \ + '"protobuf_types"' + +# Protobuf contract should be ABI 1.3 +echo "-- pb_tests (protobuf) --" +check "version is 1.3" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"version": "sysio::abi/1.3"' +check "has protobuf_types section" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"protobuf_types"' +check "protobuf_types contains proto file metadata" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"package": "test"' +check "protobuf_types contains ActData message" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"name": "ActData"' +check "protobuf_types contains ActResult message" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"name": "ActResult"' +check "single-param action type is protobuf (flattened)" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"type": "protobuf::test.ActData"' +check "action result uses protobuf type prefix" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"result_type": "protobuf::test.ActResult"' +check_absent "single-param actions have no wrapper struct" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"name": "hiproto".*"fields"' +# Multi-param protobuf action generates a wrapper struct +check "multi-param action has wrapper struct" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"name": "pbmulti"' +check "multi-param wrapper has data field" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"name": "data"' +check "multi-param wrapper has result field" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"name": "result"' +check "proto syntax is proto3" \ + "${CONTRACTS_DIR}/pb_tests.abi" \ + '"syntax": "proto3"' + +echo "" +echo "Results: ${PASS} passed, ${FAIL} failed" +[ "$FAIL" -eq 0 ] \ No newline at end of file diff --git a/tests/unit/test_contracts/CMakeLists.txt b/tests/unit/test_contracts/CMakeLists.txt index 1cc110723..3ef448567 100644 --- a/tests/unit/test_contracts/CMakeLists.txt +++ b/tests/unit/test_contracts/CMakeLists.txt @@ -20,3 +20,9 @@ configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/simple_wrong.abi ${CMAKE_CURRENT_BIN configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/capi/capi_tests.abi ${CMAKE_CURRENT_BINARY_DIR}/capi_tests.abi COPYONLY ) target_link_libraries(old_malloc_tests PUBLIC --use-freeing-malloc) + +# Protobuf test contract +add_library(pb_protos INTERFACE) +target_add_protobuf(pb_protos OUTPUT_DIRECTORY test FILES test.proto) +add_contract(pb_tests pb_tests pb_tests.cpp) +contract_use_protobuf(pb_tests pb_protos) diff --git a/tests/unit/test_contracts/pb_tests.cpp b/tests/unit/test_contracts/pb_tests.cpp new file mode 100644 index 000000000..866af4e49 --- /dev/null +++ b/tests/unit/test_contracts/pb_tests.cpp @@ -0,0 +1,38 @@ +#include +#include +#include + +namespace test { + +class [[sysio::contract]] pb_tests : public sysio::contract { +public: + using sysio::contract::contract; + + [[sysio::action]] + sysio::pb hiproto(const sysio::pb& msg) { + sysio::check(static_cast(msg.id) == 1, "validate msg.id"); + sysio::check(static_cast(msg.type) == 2, "validate msg.type"); + sysio::check(msg.note == "hello", "validate msg.note"); + + ActResult result; + result.value = zpp::bits::vint32_t(42); + result.str_value = "result_string"; + return result; + } + + [[sysio::action]] + void pbaction(const sysio::pb& msg) { + sysio::check(static_cast(msg.id) > 0, "id must be positive"); + sysio::print("Received protobuf action with id=", static_cast(msg.id)); + } + + // Multi-param action: generates a wrapper struct with two protobuf fields + [[sysio::action]] + void pbmulti(const sysio::pb& data, const sysio::pb& result) { + sysio::check(static_cast(data.id) > 0, "data.id must be positive"); + sysio::check(static_cast(result.value) > 0, "result.value must be positive"); + sysio::print("Multi-param protobuf action"); + } +}; + +} // namespace test diff --git a/tests/unit/test_contracts/test.proto b/tests/unit/test_contracts/test.proto new file mode 100644 index 000000000..658b9d0c4 --- /dev/null +++ b/tests/unit/test_contracts/test.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; +import "zpp/zpp_options.proto"; +package test; + +message ActData { + int32 id = 1; + int32 type = 2; + string note = 3; +} + +message ActResult { + int32 value = 1; + string str_value = 2; +} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index f33862b66..042bb8493 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -70,6 +70,7 @@ add_subdirectory(cc) add_subdirectory(ld) add_subdirectory(init) add_subdirectory(external) +add_subdirectory(protoc-gen-zpp) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/compiler_options.hpp.in ${CMAKE_BINARY_DIR}/compiler_options.hpp) diff --git a/tools/cc/cdt-cpp.cpp.in b/tools/cc/cdt-cpp.cpp.in index dfe94de9c..66c13e681 100644 --- a/tools/cc/cdt-cpp.cpp.in +++ b/tools/cc/cdt-cpp.cpp.in @@ -103,6 +103,15 @@ int main(int argc, const char **argv) { codegen_args.push_back(r); } + if (!opts.protobuf_dir.empty()) { + codegen_args.push_back("--protobuf-dir"); + codegen_args.push_back(opts.protobuf_dir); + } + if (!opts.protobuf_files.empty()) { + codegen_args.push_back("--protobuf-files"); + codegen_args.push_back(opts.protobuf_files); + } + for (auto& input : opts.inputs) { codegen_args.push_back(input); } diff --git a/tools/codegen/CMakeLists.txt b/tools/codegen/CMakeLists.txt index 2674abc9c..62205d3f0 100644 --- a/tools/codegen/CMakeLists.txt +++ b/tools/codegen/CMakeLists.txt @@ -1,5 +1,7 @@ set(LLVM_LINK_COMPONENTS support) +find_package(protobuf CONFIG REQUIRED) + add_executable(cdt-codegen cdt-codegen.cpp) set_property(TARGET cdt-codegen PROPERTY CXX_STANDARD 23) target_compile_options(cdt-codegen PRIVATE -fexceptions -fno-rtti) @@ -14,7 +16,7 @@ target_include_directories(cdt-codegen PRIVATE target_include_directories(cdt-codegen SYSTEM PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../jsoncons/include ) -target_link_libraries(cdt-codegen PRIVATE LLVMSupport) +target_link_libraries(cdt-codegen PRIVATE LLVMSupport protobuf::libprotobuf) add_custom_command(TARGET cdt-codegen POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ ${CMAKE_BINARY_DIR}/bin/ diff --git a/tools/codegen/cdt-codegen.cpp b/tools/codegen/cdt-codegen.cpp index c694f818f..11776c949 100644 --- a/tools/codegen/cdt-codegen.cpp +++ b/tools/codegen/cdt-codegen.cpp @@ -11,6 +11,10 @@ #include #include +#include +#include +#include + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" #include @@ -18,6 +22,16 @@ using jsoncons::ojson; +namespace gpb = google::protobuf; +namespace gpbc = google::protobuf::compiler; + +class ProtobufErrorCollector : public gpbc::MultiFileErrorCollector { + public: + void RecordError(absl::string_view filename, int line, int column, absl::string_view message) override { + std::cerr << "protobuf error: " << filename << " (" << line << ", " << column << ") " << message << std::endl; + } +}; + #ifndef CDT_VERSION #define CDT_VERSION "5.2.0" #endif @@ -151,6 +165,8 @@ static bool verbose = false; static bool suppress_ricardian_warnings = true; static bool is_wasm = false; static std::string smart_contract_trace_level; +static std::string protobuf_dir; +static std::vector protobuf_files; static int exec_subprogram(std::string prog, const std::vector& options, bool show_commands) { if (prog.size() && prog[0] != '/') { @@ -239,6 +255,16 @@ static void parse_args(int argc, const char** argv) { resource_dirs.push_back(argv[++i]); } else if (arg == "--smart-contract-trace-level" && i + 1 < argc) { smart_contract_trace_level = argv[++i]; + } else if (arg == "--protobuf-dir" && i + 1 < argc) { + protobuf_dir = argv[++i]; + } else if (arg == "--protobuf-files" && i + 1 < argc) { + // Semicolon-separated list of .proto files + std::stringstream ss(argv[++i]); + std::string item; + while (std::getline(ss, item, ';')) { + if (!item.empty()) + protobuf_files.push_back(item); + } } else if (arg[0] == '-') { std::cerr << "Unknown option: " << arg << "\n"; print_usage(argv[0]); @@ -458,6 +484,21 @@ int main(int argc, const char** argv) { } } + // Collect pb_types referenced by actions from desc files + std::set referenced_pb_types; + for (const auto& desc_name : desc_files) { + if (exists(desc_name.c_str()) && file_size(desc_name.c_str()) > 0) { + std::ifstream ifs2(desc_name); + auto desc = ojson::parse(ifs2); + ifs2.close(); + if (desc.has_key("pb_types")) { + for (auto& pb_type : desc["pb_types"].array_range()) { + referenced_pb_types.insert(pb_type.as_string()); + } + } + } + } + if (!no_abigen) { if (abi.empty()) { // No [[sysio::contract]] class found — contract may define apply() directly. @@ -467,6 +508,67 @@ int main(int argc, const char** argv) { << "', skipping ABI generation\n"; } } else { + // Embed protobuf FileDescriptorSet into ABI if protobuf files are specified + if (protobuf_files.size()) { + gpbc::DiskSourceTree source_tree; + source_tree.MapPath("", protobuf_dir); + source_tree.MapPath("", sysio::cdt::whereami::where() + "/../include"); + + ProtobufErrorCollector err_collector; + gpbc::Importer importer(&source_tree, &err_collector); + + for (auto& proto_file : protobuf_files) { + importer.Import(proto_file.c_str()); + } + + auto pool = importer.pool(); + + for (auto& type : referenced_pb_types) { + if (!pool->FindMessageTypeByName(type)) { + std::cerr << "unable to find the definition of the protobuf type: '" << type << "',\n" + "please make sure the corresponding protobuf file is correctly specified\n"; + return -1; + } + } + + gpb::FileDescriptorSet fds; + for (auto& proto_file : protobuf_files) { + auto descriptor = pool->FindFileByName(proto_file); + auto file = fds.add_file(); + descriptor->CopyTo(file); + + // Remove zpp_options.proto from dependencies (internal use only) + for (int i = file->dependency_size() - 1; i >= 0; --i) { + if (file->dependency(i).find("zpp_options.proto") != std::string::npos || + file->dependency(i).find("zpp/zpp_options.proto") != std::string::npos) { + for (int j = i; j < file->dependency_size() - 1; ++j) { + file->mutable_dependency()->SwapElements(j, j + 1); + } + file->mutable_dependency()->RemoveLast(); + } + } + } + + std::string protobuf_types_json; + auto status = gpb::util::MessageToJsonString(fds, &protobuf_types_json); + if (!status.ok()) { + std::cerr << "failed to convert protobuf types to JSON: " << status.message() << "\n"; + return -1; + } + + abi["protobuf_types"] = ojson::parse(protobuf_types_json); + + // Bump ABI version to 1.3 when protobuf_types section is present + if (abi_version_major == 1 && abi_version_minor < 3) { + abi_version_minor = 3; + abi["version"] = "sysio::abi/1.3"; + } + } else if (referenced_pb_types.size()) { + std::cerr << "protobuf types are used but no protobuf files are specified for contract " << contract_name + << ", please use `contract_use_protobuf()` cmake function to specify the protobuf files it depends on\n"; + return -1; + } + std::string filename = abi_output_path.empty() ? output_dir + "/" + contract_name + ".abi" : abi_output_path; diff --git a/tools/include/compiler_options.hpp.in b/tools/include/compiler_options.hpp.in index 8c1306b6d..abb4f94dc 100644 --- a/tools/include/compiler_options.hpp.in +++ b/tools/include/compiler_options.hpp.in @@ -356,6 +356,14 @@ static cl::opt warn_action_read_only_opt( "warn-action-read-only", cl::desc("Issue a warning if a read-only action uses a write API and continue compilation"), cl::cat(SysioCompilerToolCategory)); +static cl::opt protobuf_dir_opt( + "protobuf-dir", + cl::desc("Directory containing .proto files for protobuf contract support"), + cl::cat(SysioCompilerToolCategory)); +static cl::opt protobuf_files_opt( + "protobuf-files", + cl::desc("Semicolon-separated list of .proto files for protobuf contract support"), + cl::cat(SysioCompilerToolCategory)); /// end c/c++ options /// begin c++ options @@ -396,6 +404,8 @@ struct Options { bool has_o_opt; bool has_contract_opt; bool warn_action_read_only; + std::string protobuf_dir; + std::string protobuf_files; }; static void GetCompDefaults(std::vector& copts) { @@ -956,7 +966,6 @@ static Options CreateOptions(bool add_defaults=true) { #endif - /* TODO add some way of defaulting these to the current highest version */ int abi_version_major = 1; int abi_version_minor = 2; @@ -967,8 +976,8 @@ static Options CreateOptions(bool add_defaults=true) { } #ifndef ONLY_LD - return {output_fn, inputs, link, abigen, no_missing_ricardian_clause_opt, pp_only, pp_dir, abigen_output, abigen_contract, copts, ldopts, agopts, agresources, debug, fnative_opt, {abi_version_major, abi_version_minor}, has_o_opt, has_contract_opt, warn_action_read_only}; + return {output_fn, inputs, link, abigen, no_missing_ricardian_clause_opt, pp_only, pp_dir, abigen_output, abigen_contract, copts, ldopts, agopts, agresources, debug, fnative_opt, {abi_version_major, abi_version_minor}, has_o_opt, has_contract_opt, warn_action_read_only, std::string(protobuf_dir_opt), std::string(protobuf_files_opt)}; #else - return {output_fn, {}, link, abigen, no_missing_ricardian_clause_opt, pp_only, pp_dir, abigen_output, abigen_contract, copts, ldopts, agopts, agresources, debug, fnative_opt, {abi_version_major, abi_version_minor}, has_o_opt, has_contract_opt, warn_action_read_only}; + return {output_fn, {}, link, abigen, no_missing_ricardian_clause_opt, pp_only, pp_dir, abigen_output, abigen_contract, copts, ldopts, agopts, agresources, debug, fnative_opt, {abi_version_major, abi_version_minor}, has_o_opt, has_contract_opt, warn_action_read_only, {}, {}}; #endif } diff --git a/tools/jsoncons/include/jsoncons/json_filter.hpp b/tools/jsoncons/include/jsoncons/json_filter.hpp index 81aa6b126..4cc227e9c 100644 --- a/tools/jsoncons/include/jsoncons/json_filter.hpp +++ b/tools/jsoncons/include/jsoncons/json_filter.hpp @@ -23,8 +23,8 @@ class basic_json_filter : public basic_json_content_handler basic_json_content_handler& downstream_handler_; // noncopyable and nonmoveable - basic_json_filter(const basic_json_filter&) = delete; - basic_json_filter& operator=(const basic_json_filter&) = delete; + basic_json_filter(const basic_json_filter&) = delete; + basic_json_filter& operator=(const basic_json_filter&) = delete; public: basic_json_filter(basic_json_content_handler& handler) : downstream_handler_(handler) @@ -202,8 +202,8 @@ class basic_utf8_adaptor : public basic_json_content_handler basic_json_content_handler& downstream_handler_; // noncopyable and nonmoveable - basic_utf8_adaptor(const basic_utf8_adaptor&) = delete; - basic_utf8_adaptor& operator=(const basic_utf8_adaptor&) = delete; + basic_utf8_adaptor(const basic_utf8_adaptor&) = delete; + basic_utf8_adaptor& operator=(const basic_utf8_adaptor&) = delete; public: basic_utf8_adaptor(basic_json_content_handler& handler) : downstream_handler_(handler) diff --git a/tools/jsoncons/include/jsoncons/json_reader.hpp b/tools/jsoncons/include/jsoncons/json_reader.hpp index a60e08950..f344cb515 100644 --- a/tools/jsoncons/include/jsoncons/json_reader.hpp +++ b/tools/jsoncons/include/jsoncons/json_reader.hpp @@ -36,8 +36,8 @@ class json_utf8_other_content_handler_adapter : public json_content_handler //parse_error_handler& err_handler_; // noncopyable and nonmoveable - json_utf8_other_content_handler_adapter(const json_utf8_other_content_handler_adapter&) = delete; - json_utf8_other_content_handler_adapter& operator=(const json_utf8_other_content_handler_adapter&) = delete; + json_utf8_other_content_handler_adapter(const json_utf8_other_content_handler_adapter&) = delete; + json_utf8_other_content_handler_adapter& operator=(const json_utf8_other_content_handler_adapter&) = delete; public: json_utf8_other_content_handler_adapter() diff --git a/tools/protoc-gen-zpp/CMakeLists.txt b/tools/protoc-gen-zpp/CMakeLists.txt new file mode 100644 index 000000000..b8a3dc9b9 --- /dev/null +++ b/tools/protoc-gen-zpp/CMakeLists.txt @@ -0,0 +1,45 @@ +find_package(protobuf CONFIG REQUIRED) + +# Generate zpp_options.pb.h/cc from zpp_options.proto +set(ZPP_OPTIONS_PROTO ${CMAKE_CURRENT_SOURCE_DIR}/zpp_options.proto) +set(ZPP_OPTIONS_PB_H ${CMAKE_CURRENT_BINARY_DIR}/zpp/zpp_options.pb.h) +set(ZPP_OPTIONS_PB_CC ${CMAKE_CURRENT_BINARY_DIR}/zpp/zpp_options.pb.cc) + +set(PROTOC_PROGRAM $) + +add_custom_command( + OUTPUT ${ZPP_OPTIONS_PB_H} ${ZPP_OPTIONS_PB_CC} + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/zpp + COMMAND ${PROTOC_PROGRAM} + --proto_path=${CMAKE_CURRENT_SOURCE_DIR} + --cpp_out=${CMAKE_CURRENT_BINARY_DIR}/zpp + ${ZPP_OPTIONS_PROTO} + # protoc generates files without the zpp/ prefix in names, move them + COMMAND ${CMAKE_COMMAND} -E rename + ${CMAKE_CURRENT_BINARY_DIR}/zpp/zpp_options.pb.h + ${ZPP_OPTIONS_PB_H} + COMMAND ${CMAKE_COMMAND} -E rename + ${CMAKE_CURRENT_BINARY_DIR}/zpp/zpp_options.pb.cc + ${ZPP_OPTIONS_PB_CC} + DEPENDS ${ZPP_OPTIONS_PROTO} + COMMENT "Generating zpp_options protobuf sources" +) + +add_executable(cdt-protoc-gen-zpp protoc-gen-zpp.cpp ${ZPP_OPTIONS_PB_CC}) +target_include_directories(cdt-protoc-gen-zpp PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_link_libraries(cdt-protoc-gen-zpp PRIVATE protobuf::libprotoc protobuf::libprotobuf) + +add_custom_command(TARGET cdt-protoc-gen-zpp POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy $ ${CMAKE_BINARY_DIR}/bin/ +) + +# Also install protoc from vcpkg into CDT bin/ as cdt-protoc +add_custom_command(TARGET cdt-protoc-gen-zpp POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${PROTOC_PROGRAM} ${CMAKE_BINARY_DIR}/bin/cdt-protoc +) + +# Copy zpp_options.proto to include dir so contracts can import it +add_custom_command(TARGET cdt-protoc-gen-zpp POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/include/zpp + COMMAND ${CMAKE_COMMAND} -E copy ${ZPP_OPTIONS_PROTO} ${CMAKE_BINARY_DIR}/include/zpp/ +) diff --git a/tools/protoc-gen-zpp/protoc-gen-zpp.cpp b/tools/protoc-gen-zpp/protoc-gen-zpp.cpp new file mode 100644 index 000000000..334538c19 --- /dev/null +++ b/tools/protoc-gen-zpp/protoc-gen-zpp.cpp @@ -0,0 +1,467 @@ +/// +/// protoc-gen-zpp: protoc plugin that generates C++ structs with zpp_bits annotations +/// Ported from taurus-cdt's zpp-protoc-plugin.cpp, adapted for wire-cdt: +/// - Removed EOSIO_FRIEND_REFLECT (wire-cdt uses aggregate reflection) +/// - Changed eosio → sysio namespace references +/// +#include +#include +#include +#include +#include +#include +#include +#include +#include "zpp/zpp_options.pb.h" + +namespace gpb = google::protobuf; +namespace gpbc = google::protobuf::compiler; + +// Helper to convert absl::string_view to std::string (protobuf v6 returns string_view) +inline std::string sv2s(absl::string_view sv) { return std::string(sv); } + +std::string strip_proto(absl::string_view name) { + std::string s(name); + assert(s.size() > 6); + s.resize(s.size() - 6); + return s; +} + +bool is_keyword(const std::string& word) { + static std::unordered_set keywords = { + "NULL", "alignas", "alignof", "and", "and_eq", + "asm", "auto", "bitand", "bitor", "bool", + "break", "case", "catch", "char", "class", + "compl", "const", "constexpr", "const_cast", "continue", + "decltype", "default", "delete", "do", "double", + "dynamic_cast", "else", "enum", "explicit", "export", + "extern", "false", "float", "for", "friend", + "goto", "if", "inline", "int", "long", + "mutable", "namespace", "new", "noexcept", "not", + "not_eq", "nullptr", "operator", "or", "or_eq", + "private", "protected", "public", "register", "reinterpret_cast", + "return", "short", "signed", "sizeof", "static", + "static_assert", "static_cast", "struct", "switch", "template", + "this", "thread_local", "throw", "true", "try", + "typedef", "typeid", "typename", "union", "unsigned", + "using", "virtual", "void", "volatile", "wchar_t", + "while", "xor", "xor_eq"}; + return keywords.count(word); +} + +std::string reserve_keyword(const std::string& value) { return is_keyword(value) ? value + "_" : value; } +std::string reserve_keyword(absl::string_view value) { return reserve_keyword(std::string(value)); } + +std::string ClassName(const gpb::Descriptor* descriptor) { + const gpb::Descriptor* parent = descriptor->containing_type(); + std::string res; + if (parent) + res += ClassName(parent) + "::"; + res += sv2s(descriptor->name()); + if (descriptor->options().map_entry()) + res += "_DoNotUse"; + return reserve_keyword(res); +} + +std::string ClassName(const gpb::EnumDescriptor* enum_descriptor) { + if (enum_descriptor->containing_type() == nullptr) { + return reserve_keyword(sv2s(enum_descriptor->name())); + } else { + return ClassName(enum_descriptor->containing_type()) + "::" + sv2s(enum_descriptor->name()); + } +} + +bool IsWellKnownMessage(const gpb::FileDescriptor* file) { + static const std::unordered_set well_known_files{ + "google/protobuf/any.proto", + "google/protobuf/api.proto", + "google/protobuf/compiler/plugin.proto", + "google/protobuf/descriptor.proto", + "google/protobuf/duration.proto", + "google/protobuf/empty.proto", + "google/protobuf/field_mask.proto", + "google/protobuf/source_context.proto", + "google/protobuf/struct.proto", + "google/protobuf/timestamp.proto", + "google/protobuf/type.proto", + "google/protobuf/wrappers.proto", + }; + return well_known_files.find(sv2s(file->name())) != well_known_files.end(); +} + +std::string StringReplace(const std::string& s, const std::string& oldsub, const std::string& newsub, + bool replace_all) { + std::string res; + if (oldsub.empty()) { + return s; + } + std::string::size_type start_pos = 0; + std::string::size_type pos; + do { + pos = s.find(oldsub, start_pos); + if (pos == std::string::npos) { + break; + } + res.append(s, start_pos, pos - start_pos); + res.append(newsub); + start_pos = pos + oldsub.size(); + } while (replace_all); + res.append(s, start_pos, s.length() - start_pos); + return res; +} + +std::string DotsToColons(const std::string& name) { return StringReplace(name, ".", "::", true); } + +std::string Namespace(const std::string& package) { + if (package.empty()) + return ""; + return "::" + DotsToColons(package); +} + +std::string Namespace(const gpb::FileDescriptor* d) { + std::string ret = Namespace(sv2s(d->package())); + if (IsWellKnownMessage(d)) { + ret = StringReplace(ret, + "::google::" + "protobuf", + "::PROTOBUF_NAMESPACE_ID", false); + } + return ret; +} + +std::string QualifiedFileLevelSymbol(const gpb::FileDescriptor* file, const std::string& name) { + if (sv2s(file->package()).empty()) { + return "::" + name; + } + return Namespace(file) + "::" + name; +} + +std::string QualifiedClassName(const gpb::Descriptor* d) { return QualifiedFileLevelSymbol(d->file(), ClassName(d)); } + +std::string QualifiedClassName(const gpb::EnumDescriptor* d) { + return QualifiedFileLevelSymbol(d->file(), ClassName(d)); +} + +bool is_recursive(const gpb::Descriptor* containing_descriptor, const gpb::Descriptor* descriptor) { + while (containing_descriptor != descriptor) { + if (containing_descriptor == nullptr) + return false; + containing_descriptor = containing_descriptor->containing_type(); + } + return true; +} + +std::string full_name_to_cpp_name(absl::string_view fullname_sv) { + std::string fullname(fullname_sv); + std::string result; + size_t pos = 0; + size_t dot_pos; + while ((dot_pos = fullname.find('.', pos)) != std::string::npos) { + result += reserve_keyword(fullname.substr(pos, dot_pos - pos)); + result += "::"; + pos = dot_pos + 1; + } + result += reserve_keyword(fullname.substr(pos)); + return result; +} + +struct zpp_generator { + gpbc::CodeGeneratorResponse response; + gpb::DescriptorPool pool; + std::stringstream epilogue_strm; + + struct message_info { + std::vector> pb_map; + std::vector unique_ptr_fields; + std::map dependencies; + std::vector> messages; + std::stringstream strm; + }; + + zpp_generator(const gpbc::CodeGeneratorRequest& request) { + for (int i = 0; i < request.proto_file_size(); ++i) { + auto file_desc_proto = request.proto_file(i); + pool.BuildFile(file_desc_proto); + } + } + + void generate_enum(const gpb::EnumDescriptor* descriptor, std::stringstream& strm, std::string indent = "") { + auto name = reserve_keyword(sv2s(descriptor->name())); + strm << indent << "enum " << name << " : int {\n"; + int min_value = descriptor->value(0)->number(); + int max_value = descriptor->value(0)->number(); + + if (descriptor->value_count()) { + for (int i = 0; i < descriptor->value_count(); ++i) { + auto value = descriptor->value(i); + if (i != 0) + strm << ",\n"; + strm << indent << " " << reserve_keyword(sv2s(value->name())); + if (descriptor->options().deprecated()) + strm << " [[deprecated]]"; + strm << " = " << value->number(); + min_value = std::min(min_value, value->number()); + max_value = std::max(max_value, value->number()); + } + } + strm << "\n" << indent << "};\n\n"; + + if (min_value < -128 || max_value > 128 || (max_value - min_value) > UINT16_MAX) { + epilogue_strm << "\n" + << "template <>\n" + << "struct magic_enum::customize::enum_range<" << full_name_to_cpp_name(descriptor->full_name()) + << "> {\n" + << " static constexpr int min = " << min_value << ";\n" + << " static constexpr int max = " << max_value << ";\n" + << "};\n"; + } + } + + std::string field_type_name(const gpb::Descriptor* containing_descriptor, const gpb::FieldDescriptor* descriptor, + std::map& dependencies) { + std::string result; + bool can_be_optional = true; + + switch (descriptor->type()) { + case gpb::FieldDescriptor::TYPE_INT32: result = "zpp::bits::vint32_t"; break; + case gpb::FieldDescriptor::TYPE_INT64: result = "zpp::bits::vint64_t"; break; + case gpb::FieldDescriptor::TYPE_UINT32: result = "zpp::bits::vuint32_t"; break; + case gpb::FieldDescriptor::TYPE_UINT64: result = "zpp::bits::vuint64_t"; break; + case gpb::FieldDescriptor::TYPE_SINT32: result = "zpp::bits::vsint32_t"; break; + case gpb::FieldDescriptor::TYPE_SINT64: result = "zpp::bits::vsint64_t"; break; + case gpb::FieldDescriptor::TYPE_FIXED32: result = "uint32_t"; break; + case gpb::FieldDescriptor::TYPE_FIXED64: result = "uint64_t"; break; + case gpb::FieldDescriptor::TYPE_SFIXED32: result = "int32_t"; break; + case gpb::FieldDescriptor::TYPE_SFIXED64: result = "int64_t"; break; + case gpb::FieldDescriptor::TYPE_FLOAT: result = "float"; break; + case gpb::FieldDescriptor::TYPE_DOUBLE: result = "double"; break; + case gpb::FieldDescriptor::TYPE_BOOL: result = "bool"; break; + case gpb::FieldDescriptor::TYPE_ENUM: { + auto field = descriptor->enum_type(); + result = (containing_descriptor == field->containing_type()) ? reserve_keyword(sv2s(field->name())) + : QualifiedClassName(field); + } break; + case gpb::FieldDescriptor::TYPE_STRING: result = "std::string"; break; + case gpb::FieldDescriptor::TYPE_BYTES: result = "std::vector"; break; + case gpb::FieldDescriptor::TYPE_MESSAGE: { + auto field = descriptor->message_type(); + if (descriptor->is_map()) { + can_be_optional = false; + result = "std::map<" + field_type_name(containing_descriptor, field->map_key(), dependencies) + "," + + field_type_name(containing_descriptor, field->map_value(), dependencies) + ">"; + } else { + bool nested_type = containing_descriptor == field->containing_type(); + bool recursive = is_recursive(containing_descriptor, field); + if (nested_type) { + result = reserve_keyword(sv2s(field->name())); + } else { + result = QualifiedClassName(field); + if (!recursive) { + dependencies.try_emplace(field, containing_descriptor); + } + } + if (recursive && !descriptor->is_repeated()) { + result = "std::unique_ptr<" + result + ">"; + can_be_optional = false; + } + } + } break; + default: assert(false); + } + + if (descriptor->options().HasExtension(zpp::zpp_type)) { + result = descriptor->options().GetExtension(zpp::zpp_type); + can_be_optional = true; + } + + if (descriptor->is_repeated()) { + result = "std::vector<" + result + ">"; + if (descriptor->is_packable() && !descriptor->is_packed()) { + response.set_error("unpacked repeated field is not supported yet."); + } + } else if (can_be_optional && descriptor->options().HasExtension(zpp::zpp_optional) && + descriptor->options().GetExtension(zpp::zpp_optional)) { + result = "std::optional<" + result + ">"; + } + return result; + } + + void generate_field(const gpb::Descriptor* containing_descriptor, const gpb::FieldDescriptor* descriptor, + size_t index, message_info& info, std::string indent) { + auto field_type = field_type_name(containing_descriptor, descriptor, info.dependencies); + if (index + 1 != descriptor->number()) { + info.pb_map.emplace_back(index + 1, descriptor->number()); + } + if (field_type.find("std::unique_ptr<") == 0) + info.unique_ptr_fields.push_back(index); + + std::string attribute; + if (descriptor->options().deprecated()) + attribute = "[[deprecated]] "; + + info.strm << indent << " " << attribute << field_type << " " << reserve_keyword(sv2s(descriptor->name())) + << " = {};\n"; + } + + void generate_message(const gpb::Descriptor* descriptor, message_info& info, std::string indent = "") { + std::string message_name = reserve_keyword(sv2s(descriptor->name())); + + if (descriptor->oneof_decl_count()) { + response.set_error("oneof field is not supported yet."); + return; + } + + auto& strm = info.strm; + + strm << indent << "struct " << message_name << " {\n"; + + for (size_t i = 0; i < descriptor->enum_type_count(); ++i) + generate_enum(descriptor->enum_type(i), strm, indent + " "); + + for (size_t i = 0; i < descriptor->nested_type_count(); ++i) { + generate_message(descriptor->nested_type(i), info.dependencies, info.messages, indent + " "); + } + + for (auto [_, msg] : info.messages) { + strm << msg; + } + + for (size_t i = 0; i < descriptor->field_count(); ++i) { + generate_field(descriptor, descriptor->field(i), i, info, indent); + } + + // Emit the serialize protocol declaration for zpp_bits protobuf support + if (info.pb_map.size()) { + strm << indent << " using serialize = zpp::bits::protocol{}"; + } + strm << "}, " << descriptor->field_count() << ">;\n"; + } else { + // No field remapping needed, use default pb protocol + strm << indent << " using serialize = zpp::bits::pb_members<" << descriptor->field_count() << ">;\n"; + } + + if (descriptor->field_count()) { + if (info.unique_ptr_fields.size() == 0) + strm << indent << " bool operator == (const " << message_name << "&) const = default;\n"; + else { + strm << indent << " bool operator == (const " << message_name << "& other) const {\n"; + auto itr = info.unique_ptr_fields.begin(); + for (int i = 0; i < descriptor->field_count(); ++i) { + if (i != 0) + strm << " &&\n"; + if (itr == info.unique_ptr_fields.end() || i < *itr) + strm << indent << " " << reserve_keyword(sv2s(descriptor->field(i)->name())) << " == other." + << reserve_keyword(sv2s(descriptor->field(i)->name())); + else { + strm << indent << " ((" << reserve_keyword(sv2s(descriptor->field(i)->name())) << " == other." + << reserve_keyword(sv2s(descriptor->field(i)->name())) << ") || (*" + << reserve_keyword(sv2s(descriptor->field(i)->name())) << " == *other." + << reserve_keyword(sv2s(descriptor->field(i)->name())) << "))"; + itr++; + } + } + strm << "\n" << indent << " };\n"; + } + } + strm << indent << "}; // struct " << message_name << "\n\n"; + } + + void generate_message(const gpb::Descriptor* descriptor, + std::map& dependencies, + std::vector>& messages, std::string indent) { + message_info info; + generate_message(descriptor, info, indent); + auto depended_itr = dependencies.find(descriptor); + auto insertion_point = + depended_itr != dependencies.end() + ? std::find_if(messages.begin(), messages.end(), + [depended = depended_itr->second](auto e) { return e.first == depended; }) + : messages.end(); + messages.insert(insertion_point, std::make_pair(descriptor, info.strm.str())); + + for (auto [d, _] : info.dependencies) + dependencies.try_emplace(d, descriptor); + } + + void generate_file(const gpb::FileDescriptor* proto_file, gpbc::CodeGeneratorResponse_File* generated) { + // Check that the file uses proto3 syntax via the FileDescriptorProto + { + gpb::FileDescriptorProto fdp; + proto_file->CopyTo(&fdp); + if (fdp.syntax() != "proto3") { + response.set_error(sv2s(proto_file->name()) + ": only proto3 syntax is supported"); + return; + } + } + auto name = strip_proto(proto_file->name()) + ".pb.hpp"; + generated->set_name(name); + + std::stringstream strm; + strm << "///\n" + << "/// Generated from protoc with protoc-gen-zpp, DO NOT modify\n" + << "///\n" + << "#pragma once\n"; + + for (int i = 0; i < proto_file->dependency_count(); i++) { + const gpb::FileDescriptor* dep = proto_file->dependency(i); + auto dep_pkg = sv2s(dep->package()); + if (dep_pkg.find("google.") == 0) + continue; + if (dep_pkg.find("zpp") == 0) + continue; + if (dep->enum_type_count() > 0 || dep->message_type_count() > 0) + strm << "#include \"" << strip_proto(dep->name()) << ".pb.hpp\"\n"; + } + + strm << "#include \n" + << "// @@protoc_insertion_point(includes)\n"; + + std::string indent = ""; + std::string ns = DotsToColons(sv2s(proto_file->package())); + if (ns.size()) { + strm << "namespace " << ns << " {\n"; + } + + for (int i = 0; i < proto_file->enum_type_count(); ++i) + generate_enum(proto_file->enum_type(i), strm, indent); + + std::map dependencies; + std::vector> messages; + + for (int i = 0; i < proto_file->message_type_count() && !response.has_error(); ++i) { + generate_message(proto_file->message_type(i), dependencies, messages, indent); + } + + for (auto [_, msg] : messages) { + strm << msg; + } + + if (ns.size()) { + strm << "} // namespace " << ns << "\n"; + } + + strm << epilogue_strm.str(); + generated->set_content(strm.str()); + } + + void generate_file(std::string filename) { generate_file(pool.FindFileByName(filename), response.add_file()); } +}; + +int main(int argc, const char** argv) { + gpbc::CodeGeneratorRequest request; + request.ParseFromIstream(&std::cin); + + zpp_generator generator(request); + + for (int i = 0; i < request.file_to_generate_size(); ++i) { + generator.generate_file(request.file_to_generate(i)); + } + + generator.response.SerializePartialToOstream(&std::cout); + return 0; +} diff --git a/tools/protoc-gen-zpp/zpp_options.proto b/tools/protoc-gen-zpp/zpp_options.proto new file mode 100644 index 000000000..ac4669979 --- /dev/null +++ b/tools/protoc-gen-zpp/zpp_options.proto @@ -0,0 +1,9 @@ +syntax = "proto2"; +package zpp; + +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + optional string zpp_type = 50002; + optional bool zpp_optional = 50003; +} diff --git a/vcpkg.json b/vcpkg.json index b49f961b7..597089e52 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -3,6 +3,8 @@ "version": "5.2.0-rc1", "dependencies": [ "protobuf", + "zpp-bits", + "magic-enum", { "name": "llvm", "default-features": false,