Refactor networking

.gitmodules

@@ -10,15 +10,9 @@
 [submodule "external/zstd"]
 	path = external/zstd
 	url = https://github.com/facebook/zstd.git
-[submodule "external/nlohmann-json"]
-	path = external/nlohmann-json
-	url = https://github.com/nlohmann/json.git
-[submodule "external/oxen-libquic"]
-	path = external/oxen-libquic
-	url = https://github.com/oxen-io/oxen-libquic.git
 [submodule "external/protobuf"]
 	path = external/protobuf
 	url = https://github.com/protocolbuffers/protobuf.git
-[submodule "external/oxen-logging"]
-	path = external/oxen-logging
-	url = https://github.com/oxen-io/oxen-logging.git
+[submodule "external/lokinet"]
+	path = external/lokinet
+	url = https://github.com/oxen-io/lokinet.git

CMakeLists.txt

@@ -82,7 +82,7 @@ option(STATIC_LIBSTD "Statically link libstdc++/libgcc" ${default_static_libstd}
 option(USE_LTO "Use Link-Time Optimization" ${use_lto_default})
 
 # Provide this as an option for now because GMP and Desktop are sometimes unhappy with each other.
-option(ENABLE_ONIONREQ "Build with onion request functionality" ON)
+option(ENABLE_NETWORKING "Build with networking functionality" ON)
 
 if(USE_LTO)
   include(CheckIPOSupported)
@@ -119,7 +119,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 add_subdirectory(external)
 
-if(ENABLE_ONIONREQ)
+if(ENABLE_NETWORKING)
     if(NOT TARGET nettle::nettle)
         if(BUILD_STATIC_DEPS)
             message(FATAL_ERROR "Internal error: nettle::nettle target (expected via libquic BUILD_STATIC_DEPS) not found")

external/CMakeLists.txt

@@ -1,4 +1,5 @@
 option(SUBMODULE_CHECK "Enables checking that vendored library submodules are up to date" ON)
+
 if(SUBMODULE_CHECK)
   find_package(Git)
   if(GIT_FOUND)
@@ -26,8 +27,7 @@ if(SUBMODULE_CHECK)
     message(STATUS "Checking submodules")
     check_submodule(ios-cmake)
     check_submodule(libsodium-internal)
-    check_submodule(oxen-libquic external/oxen-logging external/oxen-encoding)
-    check_submodule(nlohmann-json)
+    check_submodule(lokinet)
     check_submodule(zstd)
     check_submodule(protobuf)
   endif()
@@ -37,28 +37,30 @@ if(NOT BUILD_STATIC_DEPS AND NOT FORCE_ALL_SUBMODULES)
   find_package(PkgConfig REQUIRED)
 endif()
 
-macro(libsession_system_or_submodule BIGNAME smallname pkgconf subdir)
-  option(FORCE_${BIGNAME}_SUBMODULE "force using ${smallname} submodule" OFF)
-  if(NOT BUILD_STATIC_DEPS AND NOT FORCE_${BIGNAME}_SUBMODULE AND NOT FORCE_ALL_SUBMODULES)
-    pkg_check_modules(${BIGNAME} ${pkgconf} IMPORTED_TARGET GLOBAL)
-  endif()
-  if(${BIGNAME}_FOUND)
-    add_library(${smallname} INTERFACE IMPORTED GLOBAL)
-    if(NOT TARGET PkgConfig::${BIGNAME} AND CMAKE_VERSION VERSION_LESS "3.21")
-      # Work around cmake bug 22180 (PkgConfig::THING not set if no flags needed)
+macro(libsession_system_or_submodule BIGNAME smallname target pkgconf subdir)
+  if(NOT TARGET ${target})
+    option(FORCE_${BIGNAME}_SUBMODULE "force using ${smallname} submodule" OFF)
+    if(NOT BUILD_STATIC_DEPS AND NOT FORCE_${BIGNAME}_SUBMODULE AND NOT FORCE_ALL_SUBMODULES)
+      pkg_check_modules(${BIGNAME} ${pkgconf} IMPORTED_TARGET GLOBAL)
+    endif()
+    if(${BIGNAME}_FOUND)
+      add_library(${smallname} INTERFACE IMPORTED GLOBAL)
+      if(NOT TARGET PkgConfig::${BIGNAME} AND CMAKE_VERSION VERSION_LESS "3.21")
+        # Work around cmake bug 22180 (PkgConfig::THING not set if no flags needed)
+      else()
+        target_link_libraries(${smallname} INTERFACE PkgConfig::${BIGNAME})
+      endif()
+      message(STATUS "Found system ${smallname} ${${BIGNAME}_VERSION}")
     else()
-      target_link_libraries(${smallname} INTERFACE PkgConfig::${BIGNAME})
+      message(STATUS "using ${smallname} submodule ${subdir}")
+      add_subdirectory(${subdir})
     endif()
-    message(STATUS "Found system ${smallname} ${${BIGNAME}_VERSION}")
-  else()
-    message(STATUS "using ${smallname} submodule")
-    add_subdirectory(${subdir})
-  endif()
-  if(TARGET ${smallname} AND NOT TARGET ${smallname}::${smallname})
-    add_library(${smallname}::${smallname} ALIAS ${smallname})
-  endif()
-  if(BUILD_STATIC_DEPS AND STATIC_BUNDLE)
+    if(NOT TARGET ${target})
+      add_library(${target} ALIAS ${smallname})
+    endif()
+    if(BUILD_STATIC_DEPS AND STATIC_BUNDLE)
       libsession_static_bundle(${smallname}::${smallname})
+    endif()
   endif()
 endmacro()
 
@@ -100,31 +102,25 @@ if(CMAKE_CROSSCOMPILING)
     endif()
 endif()
 
-set(LIBQUIC_BUILD_TESTS OFF CACHE BOOL "")
-if(ENABLE_ONIONREQ)
-  libsession_system_or_submodule(OXENQUIC quic liboxenquic>=1.3.0 oxen-libquic)
+if(ENABLE_NETWORKING)
+  set(LIBQUIC_BUILD_TESTS OFF CACHE BOOL "")
+  libsession_system_or_submodule(OXENQUIC quic oxen::quic liboxenquic>=1.6 lokinet/external/oxen-libquic)
 endif()
 
-if(NOT TARGET oxenc::oxenc)
-    # The oxenc target will already exist if we load libquic above via submodule
-    set(OXENC_BUILD_TESTS OFF CACHE BOOL "")
-    set(OXENC_BUILD_DOCS OFF CACHE BOOL "")
-    libsession_system_or_submodule(OXENC oxenc liboxenc>=1.3.0 oxen-libquic/external/oxen-encoding)
-endif()
+libsession_system_or_submodule(OXENC oxenc oxenc::oxenc liboxenc>=1.5.0 lokinet/external/oxen-libquic/external/oxen-encoding)
 
-if(NOT TARGET oxen::logging)
-    add_subdirectory(oxen-libquic/external/oxen-logging)
+libsession_system_or_submodule(OXENLOGGING oxenlogging oxen::logging liboxen-logging>=1.2.0 lokinet/external/oxen-libquic/external/oxen-logging)
+if(OXENLOGGING_FOUND)
+    # If we load oxen-logging via system lib then we won't necessarily have fmt/spdlog targets,
+    # but this script will give us them:
+    include(lokinet/external/oxen-libquic/external/oxen-logging/cmake/load_fmt_spdlog.cmake)
+
+    add_library(oxen-logging-fmt-spdlog INTERFACE)
+    target_link_libraries(oxen-logging-fmt-spdlog INTERFACE oxenlogging::oxenlogging ${OXEN_LOGGING_FMT_TARGET} ${OXEN_LOGGING_SPDLOG_TARGET})
+    add_library(oxen::logging ALIAS oxen-logging-fmt-spdlog)
 endif()
 
-oxen_logging_add_source_dir("${PROJECT_SOURCE_DIR}")
 
-# Apple xcode 15 has a completely broken std::source_location; we can't fix it, but at least we can
-# hack up the source locations to hide the path that it uses (which is the useless path to
-# oxen/log.hpp where the info/critical/etc. bodies are).
-if(APPLE AND CMAKE_CXX_COMPILER_ID STREQUAL AppleClang AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 16)
-    message(WARNING "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} is broken: filenames in logging statements will not display properly")
-    oxen_logging_add_source_dir("${CMAKE_CURRENT_SOURCE_DIR}/oxen-libquic/external/oxen-logging/include/oxen")
-endif()
 
 if(CMAKE_C_COMPILER_LAUNCHER)
   set(deps_cc "${CMAKE_C_COMPILER_LAUNCHER} ${deps_cc}")
@@ -144,12 +140,12 @@ if(APPLE)
     endforeach()
 endif()
 
-
-function(libsodium_internal_subdir)
+function(add_static_subdirectory dir)
     set(BUILD_SHARED_LIBS OFF)
-    add_subdirectory(libsodium-internal)
+    add_subdirectory(${dir} ${ARGN})
 endfunction()
-libsodium_internal_subdir()
+
+add_static_subdirectory(libsodium-internal)
 libsession_static_bundle(libsodium::sodium-internal)
 
 
@@ -163,7 +159,7 @@ set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
 set(protobuf_ABSL_PROVIDER "module" CACHE STRING "" FORCE)
 set(protobuf_BUILD_PROTOC_BINARIES OFF CACHE BOOL "")
 set(protobuf_BUILD_PROTOBUF_BINARIES ON CACHE BOOL "" FORCE)
-libsession_system_or_submodule(PROTOBUF_LITE protobuf_lite protobuf-lite>=3.21 protobuf)
+libsession_system_or_submodule(PROTOBUF_LITE protobuf_lite protobuf::libprotobuf-lite protobuf-lite>=3.21 protobuf)
 if(TARGET PkgConfig::PROTOBUF_LITE AND NOT TARGET protobuf::libprotobuf-lite)
     add_library(protobuf::libprotobuf-lite ALIAS PkgConfig::PROTOBUF_LITE)
 endif()
@@ -192,4 +188,16 @@ libsession_static_bundle(libzstd_static)
 
 set(JSON_BuildTests OFF CACHE INTERNAL "")
 set(JSON_Install ON CACHE INTERNAL "") # Required to export targets that we use
-libsession_system_or_submodule(NLOHMANN nlohmann_json nlohmann_json>=3.7.0 nlohmann-json)
+libsession_system_or_submodule(NLOHMANN nlohmann_json nlohmann_json::nlohmann_json nlohmann_json>=3.7.0 lokinet/external/nlohmann)
+
+if(ENABLE_NETWORKING)
+    set(LOKINET_FULL OFF CACHE BOOL "")
+    set(LOKINET_DAEMON OFF CACHE BOOL "")
+    set(LOKINET_NATIVE_BUILD OFF CACHE BOOL "")
+    set(LOKINET_JEMALLOC OFF CACHE BOOL "")
+
+    add_library(sodium INTERFACE)
+    target_link_libraries(sodium INTERFACE libsodium::sodium-internal)
+    add_static_subdirectory(lokinet EXCLUDE_FROM_ALL)
+    libsession_static_bundle(lokinet::liblokinet)
+endif()

include/session/file.hpp

@@ -3,6 +3,7 @@
 #include <iosfwd>
 #include <string>
 #include <string_view>
+#include <vector>
 
 // Utility functions for working with files
 
@@ -19,8 +20,8 @@ std::ofstream open_for_writing(const fs::path& filename);
 /// enabled for any failures.  This also throws if the file cannot be opened.
 std::ifstream open_for_reading(const fs::path& filename);
 
-/// Reads a (binary) file from disk into the string `contents`.
-std::string read_whole_file(const fs::path& filename);
+/// Reads a (binary) file from disk.
+std::vector<std::byte> read_whole_file(const fs::path& filename);
 
 /// Dumps (binary) string contents to disk. The file is overwritten if it already exists.
 void write_whole_file(const fs::path& filename, std::string_view contents = "");

include/session/network/backends/session_file_server.h

@@ -0,0 +1,78 @@
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "session/export.h"
+#include "session/network/session_network_types.h"
+#include "session/onionreq/builder.h"
+#include "session/platform.h"
+
+/// API: file_server/session_file_server_upload
+///
+/// Constructs a request to upload a file to the session file server.
+///
+/// Inputs:
+/// - `data` -- [in] data to upload to the file server.
+/// - `data_len` -- [in] size of the `data`.
+/// - `file_name` -- [in, optional] name of the file being uploaded. MUST be null terminated.
+/// - `request_timeout` -- [in] timeout in milliseconds to use for the request.  This won't take any
+/// pre-flight operations into account so the request will never timeout if pre-flight operations
+/// never complete.
+/// - `overall_timeout` -- [in] timeout in milliseconds to use for the request and any pre-flight
+/// operations that may need to occur (eg. path building).  This value takes presedence over
+/// `request_timeout` if provided, the request itself will be given a timeout of this value
+/// subtracting however long the pre-flight operations took.
+LIBSESSION_EXPORT session_request_params* session_file_server_upload(
+        const unsigned char* data,
+        size_t data_len,
+        const char* file_name,
+        int64_t request_timeout_ms,
+        int64_t overall_timeout_ms);
+
+/// API: network/session_file_server_download
+///
+/// Constructs a request to download a file from the session file server.
+///
+/// Inputs:
+/// - `file_id` -- [in] the id of the file to download, NULL terminated.
+/// - `request_timeout` -- [in] timeout in milliseconds to use for the request.  This won't take any
+/// pre-flight operations into account so the request will never timeout if pre-flight operations
+/// never complete.
+/// - `overall_timeout` -- [in] timeout in milliseconds to use for the request and any pre-flight
+/// operations that may need to occur (eg. path building).  This value takes presedence over
+/// `request_timeout` if provided, the request itself will be given a timeout of this value
+/// subtracting however long the pre-flight operations took.
+LIBSESSION_EXPORT session_request_params* session_file_server_download(
+        const char* file_id, int64_t request_timeout_ms, int64_t overall_timeout_ms);
+
+/// API: network/session_file_server_get_client_version
+///
+/// Constructs a request to retrieve the version information for the given platform.
+///
+/// Inputs:
+/// - `platform` -- [in] the platform to retrieve the client version for.
+/// - `ed25519_secret` -- [in] the users ed25519 secret key (used for blinded auth - 64 bytes).
+/// - `request_timeout_ms` -- [in] timeout in milliseconds to use for the request.  This won't take
+/// the path build into account so if the path build takes forever then this request will never
+/// timeout.
+/// - `request_and_path_build_timeout_ms` -- [in] timeout in milliseconds to use for the request and
+/// path build (if required).  This value takes presedence over `request_timeout_ms` if provided,
+/// the request itself will be given a timeout of this value subtracting however long it took to
+/// build the path.  A value of `0` will be ignored and `request_timeout_ms` will be used instead.
+/// - `callback` -- [in] callback to be called with the result of the request.
+/// - `ctx` -- [in, optional] Pointer to an optional context to pass through to the callback.  Set
+/// to NULL if unused.
+LIBSESSION_EXPORT session_request_params* session_file_server_get_client_version(
+        CLIENT_PLATFORM platform,
+        const unsigned char* ed25519_secret, /* 64 bytes */
+        int64_t request_timeout_ms,
+        int64_t overall_timeout_ms);
+
+#ifdef __cplusplus
+}
+#endif

include/session/network/backends/session_file_server.hpp

@@ -0,0 +1,67 @@
+#pragma once
+
+#include "session/network/key_types.hpp"
+#include "session/network/session_network_types.hpp"
+#include "session/platform.hpp"
+
+namespace session::network::file_server {
+
+/// API: file_server/upload
+///
+/// Constructs a request to upload a file to the session file server.
+///
+/// Inputs:
+/// - 'data' - [in] the data to be uploaded to a server.
+/// - `file_name` -- [in, optional] optional name to use for the file.
+/// - `request_timeout` -- [in] timeout in milliseconds to use for the request.  This won't take any
+/// pre-flight operations into account so the request will never timeout if pre-flight operations
+/// never complete.
+/// - `overall_timeout` -- [in] timeout in milliseconds to use for the request and any pre-flight
+/// operations that may need to occur (eg. path building).  This value takes presedence over
+/// `request_timeout` if provided, the request itself will be given a timeout of this value
+/// subtracting however long the pre-flight operations took.
+Request upload(
+        std::vector<unsigned char> data,
+        std::optional<std::string> file_name,
+        std::chrono::milliseconds request_timeout,
+        std::optional<std::chrono::milliseconds> overall_timeout = std::nullopt);
+
+/// API: file_server/download
+///
+/// Constructs a request to download a file from the session file server.
+///
+/// Inputs:
+/// - `file_id` -- [in] the id of the file to download.
+/// - `request_timeout` -- [in] timeout in milliseconds to use for the request.  This won't take any
+/// pre-flight operations into account so the request will never timeout if pre-flight operations
+/// never complete.
+/// - `overall_timeout` -- [in] timeout in milliseconds to use for the request and any pre-flight
+/// operations that may need to occur (eg. path building).  This value takes presedence over
+/// `request_timeout` if provided, the request itself will be given a timeout of this value
+/// subtracting however long the pre-flight operations took.
+Request download(
+        std::string file_id,
+        std::chrono::milliseconds request_timeout,
+        std::optional<std::chrono::milliseconds> overall_timeout = std::nullopt);
+
+/// API: file_server/get_client_version
+///
+/// Constructs a request to retrieve the version information for the given platform.
+///
+/// Inputs:
+/// - `platform` -- [in] the platform to retrieve the client version for.
+/// - `seckey` -- [in] the users ed25519 secret key (to generated blinded auth).
+/// - `request_timeout` -- [in] timeout in milliseconds to use for the request.  This won't take any
+/// pre-flight operations into account so the request will never timeout if pre-flight operations
+/// never complete.
+/// - `overall_timeout` -- [in] timeout in milliseconds to use for the request and any pre-flight
+/// operations that may need to occur (eg. path building).  This value takes presedence over
+/// `request_timeout` if provided, the request itself will be given a timeout of this value
+/// subtracting however long the pre-flight operations took.
+Request get_client_version(
+        Platform platform,
+        network::ed25519_seckey seckey,
+        std::chrono::milliseconds request_timeout,
+        std::optional<std::chrono::milliseconds> overall_timeout = std::nullopt);
+
+}  // namespace session::network::file_server