Speedup

.github/workflows/build-targets.yml

@@ -252,8 +252,9 @@ jobs:
 
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
+
+      - name: Parallel checkout submodules
+        run: git submodule update --init --recursive --jobs 8 --depth 1
 
       - name: Configure Linux host
         if: ${{ runner.os == 'Linux'}}

.github/workflows/build.yml

@@ -101,8 +101,9 @@ jobs:
 
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
+
+      - name: Parallel checkout submodules
+        run: git submodule update --init --recursive --jobs 8 --depth 1
 
       - name: Configure Linux host
         if: ${{ runner.os == 'Linux'}}

.gitmodules

@@ -9,21 +9,12 @@
 [submodule "ipfs-lite-cpp"]
     path = ipfs-lite-cpp
     url = ../ipfs-lite-cpp.git
-[submodule "celer-network/pb3-gen-sol"]
-    path = celer-network/pb3-gen-sol
-    url = https://github.com/celer-network/pb3-gen-sol.git
-[submodule "lmdb"]
-    path = lmdb
-    url = ../lmdb.git
 [submodule "flutter"]
     path = flutter
     url = https://github.com/flutter/flutter.git
 [submodule "curl-android-ios"]
     path = curl-android-ios
     url = https://github.com/gcesarmza/curl-android-ios.git
-[submodule "grpc"]
-    path = grpc
-    url = ../grpc
 [submodule "GSL"]
     path = GSL
     url = ../GSL.git
@@ -37,9 +28,6 @@
 	path = Boost.DI
 	url = ../Boost.DI
 	branch = cpp14
-[submodule "cpp-ipfs-http-client"]
-    path = cpp-ipfs-http-client
-    url = https://github.com/vasild/cpp-ipfs-http-client
 [submodule "ipfs-pubsub"]
     path = ipfs-pubsub
     url = ../ipfs-pubsub.git
@@ -52,9 +40,6 @@
 [submodule "rapidjson"]
     path = rapidjson
     url = ../rapidjson.git
-[submodule "gnostic"]
-    path = gnostic
-    url = https://github.com/googleapis/gnostic
 [submodule "rocksdb"]
     path = rocksdb
     url = ../rocksdb.git
@@ -117,9 +102,6 @@
 [submodule "AsyncIOManager"]
     path = AsyncIOManager
     url = ../AsyncIOManager.git
-[submodule "cryptopp-cmake"]
-    path = cryptopp-cmake
-    url = https://github.com/abdes/cryptopp-cmake.git
 [submodule "stb"]
     path = stb
     url = https://github.com/nothings/stb.git
@@ -133,8 +115,11 @@
     path = SQLiteModernCpp
     url = ../sqlite_modern_cpp.git
 [submodule "json"]
-    path = json
-    url = git@github.com:nlohmann/json.git
+	path = json
+	url = https://github.com/nlohmann/json.git
 [submodule "snappy"]
     path = snappy
     url = https://github.com/google/snappy.git
+[submodule "protobuf"]
+	path = protobuf
+	url = https://github.com/protocolbuffers/protobuf.git

build/CommonTargets.CMake

@@ -15,7 +15,7 @@ if(CMAKE_CROSSCOMPILING)
 
     ExternalProject_Add(protobuf_host
         PREFIX ${PROTOBUF_HOST_PREFIX}
-        SOURCE_DIR "${THIRDPARTY_DIR}/grpc/third_party/protobuf"
+        SOURCE_DIR "${THIRDPARTY_DIR}/protobuf"
         CMAKE_ARGS
         -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
         CMAKE_CACHE_ARGS
@@ -29,17 +29,12 @@ if(CMAKE_CROSSCOMPILING)
         ${_PROTOBUF_HOST_EXTRA_CACHE_ARGS}
     )
 else()
-    set(PROTOBUF_HOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/grpc")
-    set(PROTOBUF_TARGET grpc)
-endif()
-
-if(MSVC)
-    set(Protobuf_HOST_DIR "${PROTOBUF_HOST_PREFIX}/cmake")
-else()
-    set(Protobuf_HOST_DIR "${PROTOBUF_HOST_PREFIX}/lib/cmake/protobuf")
+    set(PROTOBUF_HOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/protobuf")
+    set(PROTOBUF_TARGET protobuf)
 endif()
 
 set(absl_HOST_DIR "${PROTOBUF_HOST_PREFIX}/lib/cmake/absl")
+set(Protobuf_HOST_DIR "${PROTOBUF_HOST_PREFIX}/lib/cmake/protobuf")
 set(utf8_range_HOST_DIR "${PROTOBUF_HOST_PREFIX}/lib/cmake/utf8_range")
 
 set(PROTOBUF_HOST_INCLUDE_DIR "${PROTOBUF_HOST_PREFIX}/include")
@@ -76,76 +71,41 @@ set(_FINDPACKAGE_CARES_CONFIG_DIR "${INSTALL_DIR}/lib/cmake/c-ares")
 set(_FINDPACKAGE_CARES_LIBRARY_DIR "${INSTALL_DIR}/lib")
 set(_FINDPACKAGE_CARES_INCLUDE_DIR "${INSTALL_DIR}/include")
 
-# gRPC
-ExternalProject_Add(grpc
-    PREFIX grpc
-    SOURCE_DIR "${THIRDPARTY_DIR}/grpc"
+# Protobuf
+ExternalProject_Add(protobuf
+    PREFIX protobuf
+    SOURCE_DIR "${THIRDPARTY_DIR}/protobuf"
     CMAKE_ARGS
     -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
     CMAKE_CACHE_ARGS
     ${_CMAKE_COMMON_CACHE_ARGS}
     -DABSL_ENABLE_INSTALL:BOOL=ON
     -DBUILD_SHARED_LIBS:BOOL=OFF
     -DBUILD_TESTING:BOOL=OFF
-    -Dc-ares_DIR:PATH=${_FINDPACKAGE_CARES_CONFIG_DIR}
-    -DgRPC_BUILD_CODEGEN:BOOL=OFF
-    -DgRPC_BUILD_CSHARP_EXT:BOOL=OFF
-    -DgRPC_BUILD_GRPC_CPP_PLUGIN:BOOL=OFF
-    -DgRPC_BUILD_GRPC_CSHARP_PLUGIN:BOOL=OFF
-    -DgRPC_BUILD_GRPC_NODE_PLUGIN:BOOL=OFF
-    -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN:BOOL=OFF
-    -DgRPC_BUILD_GRPC_PHP_PLUGIN:BOOL=OFF
-    -DgRPC_BUILD_GRPC_PYTHON_PLUGIN:BOOL=OFF
-    -DgRPC_BUILD_GRPC_RUBY_PLUGIN:BOOL=OFF
-    -DgRPC_CARES_PROVIDER:STRING=package
-    -DgRPC_INSTALL:BOOL=ON
-    -DgRPC_SSL_PROVIDER:STRING=package
-    -DgRPC_ZLIB_PROVIDER:STRING=package
-    -Dprotobuf_ABSL_PROVIDER:STRING=package
     -Dprotobuf_MODULE_COMPATIBLE:BOOL=ON
     -Dprotobuf_MSVC_STATIC_RUNTIME:BOOL=ON
-    -DRE2_BUILD_TESTING:BOOL=OFF
-    -DZLIB_ROOT:PATH=${ZLIB_ROOT}
-    -DZLIB_USE_STATIC_LIBS:BOOL=ON
-    -DABSL_PROPAGATE_CXX_STD:BOOL=ON
-    ${_OPENSSL_CACHE_ARGS}
-    DEPENDS openssl zlib cares
 )
-
-ExternalProject_Get_Property(grpc INSTALL_DIR)
-
-# absl
+ExternalProject_Get_Property(protobuf INSTALL_DIR)
 set(absl_DIR "${INSTALL_DIR}/lib/cmake/absl")
-
-# utf8_range
 set(utf8_range_DIR "${INSTALL_DIR}/lib/cmake/utf8_range")
-
-# Protobuf
-if(MSVC)
-    set(Protobuf_DIR "${INSTALL_DIR}/cmake")
-else()
-    set(Protobuf_DIR "${INSTALL_DIR}/lib/cmake/protobuf")
-endif()
-
-set(Protobuf_LIBRARIES "${INSTALL_DIR}/lib")
-set(Protobuf_INCLUDE_DIR "${INSTALL_DIR}/include")
+set(Protobuf_DIR "${INSTALL_DIR}/lib/cmake/protobuf")
 
 # Snappy for RocksDB compression
 ExternalProject_Add(snappy
-        PREFIX snappy
-        SOURCE_DIR "${THIRDPARTY_DIR}/snappy"
-        CMAKE_ARGS
-        CMAKE_CACHE_ARGS
-        -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
-        -DFAIL_ON_WARNINGS:BOOL=OFF
-        -DSNAPPY_BUILD_TESTS:BOOL=OFF
-        -DSNAPPY_BUILD_BENCHMARKS:BOOL=OFF
-        ${_CMAKE_COMMON_CACHE_ARGS}
-        # these need to be at the end to override previous cached args line
-        -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}\ -DNDEBUG
-        -DCMAKE_C_FLAGS_DEBUG:STRING=${CMAKE_C_FLAGS_DEBUG}\ -DNDEBUG
-        BUILD_BYPRODUCTS
-        "${CMAKE_CURRENT_BINARY_DIR}/snappy/lib/libsnappy.a"
+    PREFIX snappy
+    SOURCE_DIR "${THIRDPARTY_DIR}/snappy"
+    CMAKE_ARGS
+    CMAKE_CACHE_ARGS
+    -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+    -DFAIL_ON_WARNINGS:BOOL=OFF
+    -DSNAPPY_BUILD_TESTS:BOOL=OFF
+    -DSNAPPY_BUILD_BENCHMARKS:BOOL=OFF
+    ${_CMAKE_COMMON_CACHE_ARGS}
+    # these need to be at the end to override previous cached args line
+    -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}\ -DNDEBUG
+    -DCMAKE_C_FLAGS_DEBUG:STRING=${CMAKE_C_FLAGS_DEBUG}\ -DNDEBUG
+    BUILD_BYPRODUCTS
+    "${CMAKE_CURRENT_BINARY_DIR}/snappy/lib/libsnappy.a"
 )
 
 set(_FINDPACKAGE_SNAPPY_DIR "${CMAKE_CURRENT_BINARY_DIR}/snappy/lib/cmake/Snappy")
@@ -368,17 +328,12 @@ ExternalProject_Add(libp2p
     ${_OPENSSL_CACHE_ARGS}
     ${_CMAKE_COMMON_CACHE_ARGS}
     -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}\ -I\ ${_FINDPACKAGE_GSL_INCLUDE_DIR}\ -I\ ${_FINDPACKAGE_SQLITE3_INCLUDE_DIR}
-    DEPENDS Boost Microsoft.GSL openssl cares ${PROTOBUF_TARGET} fmt yaml-cpp soralog tsl_hat_trie Boost.DI SQLiteModernCpp grpc
+    DEPENDS Boost Microsoft.GSL openssl cares ${PROTOBUF_TARGET} fmt yaml-cpp soralog tsl_hat_trie Boost.DI SQLiteModernCpp protobuf
 )
 set(_FINDPACKAGE_libp2p_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}/libp2p/lib/cmake/libp2p")
 set(_FINDPACKAGE_libp2p_LIBRARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/libp2p/lib")
 set(_FINDPACKAGE_LIBP2P_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/libp2p/include")
 
-#if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
-#    set(IS_APPLE_PLATFORM ON)
-#elseif(CMAKE_SYSTEM_NAME STREQUAL "iOS")
-#    set(IS_APPLE_PLATFORM ON)
-#endif()
 if(NOT ANDROID)
     # Vulkan-Headers
     ExternalProject_Add(
@@ -412,25 +367,25 @@ if(NOT ANDROID)
 endif()
 
 if(NOT APPLE OR (APPLE AND CMAKE_OSX_SYSROOT MATCHES "iPhoneOS"))
-ExternalProject_Add(MNN
-    PREFIX MNN
-    SOURCE_DIR "${THIRDPARTY_DIR}/MNN"
-    CMAKE_CACHE_ARGS
-    -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
-    -DMNN_BUILD_SHARED_LIBS:BOOL=OFF
-    -DMNN_BUILD_TESTS:BOOL=OFF
-    -DMNN_BUILD_TOOLS:BOOL=OFF
-    -DMNN_BUILD_PROTOBUFFER:BOOL=OFF
-    -DMNN_VULKAN:BOOL=ON
-    -DMNN_WIN_RUNTIME_MT:BOOL=ON
-    ${_CMAKE_COMMON_CACHE_ARGS}
-    ${_MNN_EXTRA_PARAM}
-    DEPENDS ${_MNN_DEPENDS} ${vulkanTarget}
-)
+    ExternalProject_Add(MNN
+        PREFIX MNN
+        SOURCE_DIR "${THIRDPARTY_DIR}/MNN"
+        CMAKE_CACHE_ARGS
+        -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+        -DMNN_BUILD_SHARED_LIBS:BOOL=OFF
+        -DMNN_BUILD_TESTS:BOOL=OFF
+        -DMNN_BUILD_TOOLS:BOOL=OFF
+        -DMNN_BUILD_PROTOBUFFER:BOOL=OFF
+        -DMNN_VULKAN:BOOL=ON
+        -DMNN_WIN_RUNTIME_MT:BOOL=ON
+        ${_CMAKE_COMMON_CACHE_ARGS}
+        ${_MNN_EXTRA_PARAM}
+        DEPENDS ${_MNN_DEPENDS} ${vulkanTarget}
+    )
 
-set(_FINDPACKAGE_MNN_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}/MNN/lib/cmake/MNN")
-set(_FINDPACKAGE_MNN_LIBRARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/MNN/lib")
-set(_FINDPACKAGE_MNN_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/MNN/include")
+    set(_FINDPACKAGE_MNN_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}/MNN/lib/cmake/MNN")
+    set(_FINDPACKAGE_MNN_LIBRARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/MNN/lib")
+    set(_FINDPACKAGE_MNN_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/MNN/include")
 endif()
 
 # ipfs-lite-cpp
@@ -463,7 +418,7 @@ ExternalProject_Add(ipfs-lite-cpp
     ${_BOOST_CACHE_ARGS}
     ${_OPENSSL_CACHE_ARGS}
     -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}\ -I\ ${_FINDPACKAGE_GSL_INCLUDE_DIR}
-    DEPENDS Microsoft.GSL spdlog tsl_hat_trie libp2p grpc ${PROTOBUF_TARGET} rocksdb Boost.DI
+    DEPENDS Microsoft.GSL spdlog tsl_hat_trie libp2p protobuf ${PROTOBUF_TARGET} rocksdb Boost.DI
 )
 set(_FINDPACKAGE_IPFS_LITE_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}/ipfs-lite-cpp/lib/cmake/ipfs-lite-cpp")
 set(_FINDPACKAGE_IPFS_LITE_LIBRARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/ipfs-lite-cpp/lib")
@@ -616,8 +571,8 @@ ExternalProject_Add(protobuf-plugin
     INSTALL_DIR protobuf-plugin
     CMAKE_CACHE_ARGS
     -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
-    -Dabsl_DIR:PATH=${absl_HOST_DIR}
     -DCMAKE_BUILD_TYPE:STRING=Release
+    -Dabsl_DIR:PATH=${absl_HOST_DIR}
     -DProtobuf_DIR:PATH=${Protobuf_HOST_DIR}
     -Dprotobuf_MODULE_COMPATIBLE:BOOL=ON
     -DTW_USE_EXTERNAL_PROTOC:BOOL=TRUE
@@ -657,8 +612,8 @@ ExternalProject_Add(wallet-core
     # doesn't short-circuit a second config (e.g. Release after Debug) and skip
     # building the correct config's artifact.
     PATCH_COMMAND ${CMAKE_COMMAND} -E rm -f "${THIRDPARTY_DIR}/wallet-core/rust/target/libwallet_core_rs.a"
-        COMMAND ${_WALLET_CORE_GENERATE_SCRIPT} --target-os=${_WALLET_CORE_ARCH} --arch-abi=${WALLET_CORE_ARCH_ABI} --protobuf-dir=${PROTOBUF_HOST_PREFIX} --out-dir=<INSTALL_DIR> --plugin-dir=${ProtobufPlugin_DIR}/bin --build-type=$<CONFIG>
-    DEPENDS Boost grpc openssl protobuf-plugin
+    COMMAND ${_WALLET_CORE_GENERATE_SCRIPT} --target-os=${_WALLET_CORE_ARCH} --arch-abi=${WALLET_CORE_ARCH_ABI} --protobuf-dir=${PROTOBUF_HOST_PREFIX} --out-dir=<INSTALL_DIR> --plugin-dir=${ProtobufPlugin_DIR}/bin --build-type=$<CONFIG>
+    DEPENDS Boost protobuf openssl protobuf-plugin
     BUILD_BYPRODUCTS
     "${CMAKE_CURRENT_BINARY_DIR}/wallet-core/lib/libTrustWalletCore.a"
 )
@@ -718,11 +673,8 @@ ExternalProject_Add(AsyncIOManager
     ${_BOOST_CACHE_ARGS}
     ${_OPENSSL_CACHE_ARGS}
     ${_CMAKE_COMMON_CACHE_ARGS}
-    DEPENDS libssh2 openssl ipfs-lite-cpp ipfs-bitswap-cpp grpc ${PROTOBUF_TARGET} Boost MNN
+    DEPENDS libssh2 openssl ipfs-lite-cpp ipfs-bitswap-cpp protobuf ${PROTOBUF_TARGET} Boost MNN
 )
-set(_FINDPACKAGE_ASIOMGR_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}/AsyncIOManager/lib/cmake/AsyncIOManager")
-set(_FINDPACKAGE_ASIOMGR_LIBRARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/AsyncIOManager/lib")
-set(_FINDPACKAGE_ASIOMGR_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/AsyncIOManager/include")
 
 # gnus_upnp
 ExternalProject_Add(gnus_upnp
@@ -741,25 +693,19 @@ ExternalProject_Add(gnus_upnp
     ${_OPENSSL_CACHE_ARGS}
     DEPENDS Boost fmt spdlog tsl_hat_trie yaml-cpp
 )
-set(_FINDPACKAGE_GNUS_UPNP_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}/gnus_upnp/lib/cmake/gnus_upnp")
-set(_FINDPACKAGE_GNUS_UPNP_LIBRARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/gnus_upnp/lib")
-set(_FINDPACKAGE_GNUS_UPNP_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/gnus_upnp/include")
 
-#json.hpp
+# Nlohmann JSON
 ExternalProject_Add(json
     PREFIX json
     SOURCE_DIR "${THIRDPARTY_DIR}/json"
     CMAKE_CACHE_ARGS
     -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
-	-DJSON_BuildTests:BOOL=OFF
+    -DJSON_BuildTests:BOOL=OFF
     ${_CMAKE_COMMON_CACHE_ARGS}
     ${_BOOST_CACHE_ARGS}
     ${_OPENSSL_CACHE_ARGS}
     DEPENDS Boost
 )
-set(_FINDPACKAGE_JSON_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}/json/lib/cmake/valijson")
-set(_FINDPACKAGE_JSON_LIBRARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/json/lib")
-set(_FINDPACKAGE_JSON_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/json/include")
 
 # jsonrpc-lean
 install(