VER: Release 0.7.0

databento · Apr 28, 2023 · 011fe5e · 011fe5e
2 parents 1a5d5f6 + c96a673
commit 011fe5e
Show file tree

Hide file tree

Showing 58 changed files with 1,676 additions and 558 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -32,7 +32,9 @@ jobs:
             -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} \
             -DDATABENTO_ENABLE_UNIT_TESTING=1 \
             -DDATABENTO_ENABLE_EXAMPLES=1 \
-            -DDATABENTO_ENABLE_CLANG_TIDY=1
+            -DDATABENTO_ENABLE_CLANG_TIDY=1 \
+            -DDATABENTO_ENABLE_ASAN=1 \
+            -DDATABENTO_ENABLE_UBSAN=1
       - name: CMake build
         run: cmake --build build
       - name: Unit tests
@@ -54,6 +56,8 @@ jobs:
             -GNinja \
             -DDATABENTO_ENABLE_UNIT_TESTING=1 \
             -DDATABENTO_ENABLE_EXAMPLES=1 \
+            -DDATABENTO_ENABLE_ASAN=1 \
+            -DDATABENTO_ENABLE_UBSAN=1 \
             -DOPENSSL_ROOT_DIR=$(brew --prefix openssl@3)
       - name: CMake build
         run: cmake --build build

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,29 @@
 # Changelog
 
+## 0.7.0 - 2023-04-28
+- Added initial support for live data with `LiveBlocking` and `LiveThreaded` clients
+- Added support for statistics schema
+- Added `SystemMsg` and `ErrorMsg` records for use in live data
+- Added `strike_price`, `strike_price_currency`, and `instrument_class` to `InstrumentDefMsg`
+- Renamed `BatchJob.cost` to `cost_usd` and value now expressed as US dollars
+- Added `FixedPx` helper class for formatting fixed prices
+- Added configurable log receiver `ILogReceiver`
+- Added `instrument_class`, `strike_price`, and `strike_price_currency` to definition schema
+- Added additional `condition` variants for `DatasetConditionDetail` (degraded, pending, missing)
+- Added additional member `last_modified_date` to `DatasetConditionDetail` Added `has_mixed_schema`, `has_mixed_stype_in`, and `ts_out` to `Metadata` to support live data
+- Removed `related` and `related_security_id` from `InstrumentDefMsg`
+- Renamed `SType::ProductId` to `SType::InstrumentId` and `SType::Native` to `SType::RawSymbol`
+- Renamed `RecordHeader::product_id` to `instrument_id`
+- Renamed `InstrumentDefMsg::symbol` to `raw_symbol`
+- Renamed `SymbolMapping::native_symbol` to `raw_symbol`
+- Deprecated `SType::Smart` to split into `SType::Parent` and `SType::Continuous`
+- Changed `expiration` and `action` type to `UnixNanos`
+- Changed some fields to enums in `InstrumentDefMsg`
+- Added optional `compression` parameter to `BatchSubmitJob`
+- Fixed parsing of `BatchSubmitJob` response
+- Fixed invalid read in `DbnDecoder`
+- Fixed memory leak in `TryCreateDir`
+
 ## 0.6.1 - 2023-03-28
 - Fixed Zstd decoding of files with multiple frames
 - Removed usage of unreliable `std::ifstream::readsome`

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 3.14)
 # Project details
 #
 
-project("databento" VERSION 0.6.1 LANGUAGES CXX)
+project("databento" VERSION 0.7.0 LANGUAGES CXX)
 string(TOUPPER ${PROJECT_NAME} PROJECT_NAME_UPPERCASE)
 
 #
@@ -103,6 +103,7 @@ message(STATUS "Added all header and implementation files.\n")
 target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_11)
 include(cmake/CompilerWarnings.cmake)
 set_project_warnings(${PROJECT_NAME})
+include(cmake/Sanitizers.cmake)
 
 # Ensure std::string debug info is included
 if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -136,19 +137,20 @@ else()
   FetchContent_MakeAvailable(json)
 endif()
 # cpp-httplib
+set(httplib_version 0.11.4)
 if(${PROJECT_NAME_UPPERCASE}_USE_EXTERNAL_HTTPLIB)
-  find_package(httplib 0.11.2 REQUIRED)
+  find_package(httplib ${httplib_version} REQUIRED)
 else()
   if(CMAKE_VERSION VERSION_LESS 3.24)
     FetchContent_Declare(
       httplib
-      URL https://github.com/yhirose/cpp-httplib/archive/refs/tags/v0.11.2.tar.gz
+      URL https://github.com/yhirose/cpp-httplib/archive/refs/tags/v${httplib_version}.tar.gz
     )
   else()
     # DOWNLOAD_EXTRACT_TIMESTAMP added in 3.24
     FetchContent_Declare(
       httplib
-      URL https://github.com/yhirose/cpp-httplib/archive/refs/tags/v0.11.2.tar.gz
+      URL https://github.com/yhirose/cpp-httplib/archive/refs/tags/v${httplib_version}.tar.gz
       DOWNLOAD_EXTRACT_TIMESTAMP TRUE
     )
   endif()

diff --git a/README.md b/README.md
@@ -4,52 +4,19 @@
 [![license](https://img.shields.io/github/license/databento/databento-cpp?color=blue)](./LICENSE)
 
 The official C++ client library for [Databento](https://databento.com).
-The client supports both streaming live and historical data through similar interfaces.
-
-**Please note:** this client currently only supports historical data and is under active development as Databento prepares to launch live data.
+The client supports both streaming real-time and historical market data through similar interfaces.
 
 ## Usage
 
-A simple program that fetches a day's worth of historical trades for all ES symbols and prints it looks like this:
-
-```cpp
-#include <databento/historical.hpp>
-#include <iostream>
-
-using namespace databento;
-
-static constexpr auto kApiKey = "YOUR_API_KEY";
-
-int main() {
-  auto client = HistoricalBuilder{}.SetKey(kApiKey).Build();
-  client.TimeseriesGetRange("GLBX.MDP3", "2022-06-10", "2022-06-11", {"ES"},
-                          Schema::Trades, SType::Smart, SType::ProductId, {},
-                          {}, [](const Record& record) {
-                            const auto& trade_msg = record.Get<TradeMsg>();
-                            std::cout << trade_msg << '\n';
-                            return KeepGoing::Continue;
-                          });
-}
-```
-
-To run this program, replace `YOUR_API_KEY` with an actual API key.
+The minimum C++ standard is C++11 and the minimum CMake version is 3.14.
 
-Additional example standalone executables are provided in the [examples](./examples) directory.
-These examples can be compiled by enabling the cmake option `DATABENTO_ENABLE_EXAMPLES` with `-DDATABENTO_ENABLE_EXAMPLES=1` during the configure step.
-
-### Documentation
-
-More detailed examples and the full API documentation can be found on the [Databento doc site](https://docs.databento.com/getting-started).
+### Integration
 
-## Integration
+The easiest way to use our library is by embedding it with [CMake FetchContent](https://cmake.org/cmake/help/v3.11/module/FetchContent.html).
+Your `CMakeLists.txt` should look something like the following:
 
-databento-cpp can be integrated into C++ projects in a couple of different ways.
-
-### Embedded
-
-The easiest way to integrate databento-cpp into your project is using CMake's [FetchContent](https://cmake.org/cmake/help/latest/module/FetchContent.html).
-The minimum supported CMake version is 3.14.
 ```cmake
+# CMakeLists.txt
 include(FetchContent)
 
 FetchContent_Declare(
@@ -63,9 +30,10 @@ add_library(my_library)
 target_link_libraries(my_library PRIVATE databento::databento)
 ```
 
-### System
+Alternatively, you can clone the source code from GitHub [here](https://github.com/databento/databento-cpp).
+
+To install the library at the system level, build and install it with the following:
 
-To install databento-cpp at the system level, clone the repo, build, and install with CMake.
 ```sh
 git clone https://github.com/databento/databento-cpp
 cd databento-cpp
@@ -78,17 +46,17 @@ cmake --build build --target databento
 cmake --install build
 ```
 
-Then in your project's `CMakeLists.txt`, add the following:
+In your project's `CMakeLists.txt`, add the following:
+
 ```cmake
-find_package(databento 0.6.1 REQUIRED)
-add_library(my_library)
-target_link_libraries(my_library PRIVATE databento::databento)
+# CMakeLists.txt
+find_package(databento REQUIRED)
+target_link_libraries(your_target PRIVATE databento::databento)
 ```
 
-## Requirements
+### Dependencies
 
-The minimum C++ standard is C++11 and CMake 3.14.
-The library has the following dependencies:
+You'll need to ensure the following dependencies are installed:
 - [OpenSSL](https://www.openssl.org/)
 - [Libcrypto](https://www.openssl.org/docs/man3.0/man7/crypto.html)
 - [Zstandard (zstd)](https://github.com/facebook/zstd)
@@ -99,75 +67,85 @@ By default, cpp-httplib and nlohmann\_json are downloaded by CMake as part of th
 If you would like to use a local version of these libraries, enable the CMake flag
 `DATABENTO_ENABLE_EXTERNAL_HTTPLIB` or `DATABENTO_ENABLE_EXTERNAL_JSON`.
 
-The other two dependencies (zstd and OpenSSL) are available in most package managers.
-For example, on Ubuntu and Debian:
-```sh
-sudo apt install libssl-dev libzstd-dev
-```
+### Live
 
-## Building locally
+Real-time and intraday replay is provided through the Live clients.
+Here is a simple program that fetches 10 seconds of trades for all ES mini futures:
 
-databento-cpp uses [CMake](https://cmake.org/) as its build system, with a minimum version of 3.14.
-Building with `cmake` is a two step process: first configuring, then building.
-```sh
-cmake -S . -B build  # configure
-cmake --build build -- -j $(nproc)  # build all targets with all cores
-```
+```cpp
+#include <chrono>
+#include <databento/live.hpp>
+#include <iostream>
+#include <string>
+#include <thread>
+#include <unordered_map>
 
-### Testing
+using namespace databento;
 
-Tests are located in the `test` directory.
-They're written using [GoogleTest (gtest)](https://github.com/google/googletest).
-The test target is `databentoTests` and can be build and run with the following commands:
-```sh
-cmake -S . -B build -DDATABENTO_ENABLE_UNIT_TESTING=1  # configure
-cmake --build build --target databentoTests  # build
-build/test/databentoTests  # run
+int main() {
+  std::unordered_map<std::uint32_t, std::string> symbol_mappings;
+
+  auto client = LiveBuilder{}
+                    .SetKey("$YOUR_API_KEY")
+                    .SetDataset("GLBX.MDP3")
+                    .BuildThreaded();
+
+  auto handler = [&symbol_mappings](const Record& rec) {
+    if (rec.Holds<TradeMsg>()) {
+      auto trade = rec.Get<TradeMsg>();
+      std::cout << "Received trade for "
+                << symbol_mappings[trade.hd.instrument_id] << ':'
+                << trade << '\n';
+    } else if (rec.Holds<SymbolMappingMsg>()) {
+      auto mapping = rec.Get<SymbolMappingMsg>();
+      symbol_mappings[mapping.hd.instrument_id] =
+          mapping.stype_out_symbol.data();
+    }
+    return KeepGoing::Continue;
+  };
+
+  client.Subscribe({"ES.FUT"}, Schema::Trades, SType::Parent);
+  client.Start(handler);
+  std::this_thread::sleep_for(std::chrono::seconds{10});
+  return 0;
+}
 ```
+To run this program, replace `$YOUR_API_KEY` with an actual API key.
 
-By default, it's assumed google test is installed already, if instead you'd like CMake to
-download it for you, disable the `DATABENTO_USE_EXTERNAL_GTEST` flag:
-```sh
-cmake -S . -B build \
-  -DDATABENTO_ENABLE_UNIT_TESTING=1 \
-  -DDATABENTO_USE_EXTERNAL_GTEST=0
-```
+### Historical
 
-### Formatting
+Here is a simple program that fetches 10 minutes worth of historical trades for the entire CME Globex market:
 
-databento-cpp uses [`clang-format`](https://clang.llvm.org/docs/ClangFormat.html) with Google's style.
-`clang-format` usually comes installed with [clang](https://clang.llvm.org/).
-You can run `clang-format` against all the files in `databento-cpp` with the following command:
-```sh
-cmake --build build --target clang-format
-```
+```cpp
+#include <databento/constants.hpp>
+#include <databento/historical.hpp>
+#include <iostream>
 
-### Linting
+using namespace databento;
 
-databento-cpp uses Clang-Tidy and Cppcheck for linting and detecting potential mistakes.
-Both can be enabled to run as part of compilation through CMake flags.
-```sh
-cmake -S . -B build -DDATABENTO_ENABLE_CLANG_TIDY=1 -DDATABENTO_ENABLE_CPPCHECK=1
-cmake --build build   # compiles code, and runs Clang-Tidy and Cppcheck
+int main() {
+  auto client =
+      HistoricalBuilder{}.SetKey("$YOUR_API_KEY").Build();
+  auto print_trades = [](const Record& record) {
+    const auto& trade_msg = record.Get<TradeMsg>();
+    std::cout << trade_msg << '\n';
+    return KeepGoing::Continue;
+  };
+  client.TimeseriesGetRange(
+      "GLBX.MDP3", "2022-06-10T14:30", "2022-06-10T14:40",
+      kAllSymbols, Schema::Trades, SType::RawSymbol,
+      SType::InstrumentId, {}, {}, print_trades);
+}
 ```
 
-### macOS
+To run this program, replace `$YOUR_API_KEY` with an actual API key.
 
-To setup OpenSSL and zstd, run the following:
-```sh
-brew install openssl zstd
-# Add it to the PATH so cmake can find it
-export "$PATH:$HOMEBREW_PREFIX/opt/openssl/bin"
-```
+Additional example standalone executables are provided in the [examples](./examples) directory.
+These examples can be compiled by enabling the cmake option `DATABENTO_ENABLE_EXAMPLES` with `-DDATABENTO_ENABLE_EXAMPLES=1` during the configure step.
 
-For linting on macOS, the best way to install clang-tidy and clang-format is to install all of LLVM
-and symlink the binaries to some place in your `PATH`.
-```sh
-brew install llvm
-ln -s $(brew --prefix llvm)/bin/clang-tidy $HOME/.local/bin/
-ln -s $(brew --prefix llvm)/bin/clang-format $HOME/.local/bin/
-ln -s $(brew --prefix llvm)/bin/run-clang-format $HOME/.local/bin/
-```
+## Documentation
+
+You can find more detailed examples and the full API documentation on the [Databento doc site](https://docs.databento.com/getting-started?historical=cpp&live=cpp).
 
 ## License
 

diff --git a/cmake/CompilerWarnings.cmake b/cmake/CompilerWarnings.cmake
@@ -59,13 +59,9 @@ function(set_project_warnings project_name)
       -Wdouble-promotion # warn if float is implicit promoted to double
       -Wformat=2 # warn on security issues around functions that format output
                  # (ie printf)
+      -Wimplicit-fallthrough # warn on switch labels without break
   )
 
-  if (${PROJECT_NAME}_WARNINGS_AS_ERRORS)
-    set(CLANG_WARNINGS ${CLANG_WARNINGS} -Werror)
-    set(MSVC_WARNINGS ${MSVC_WARNINGS} /WX)
-  endif()
-
   set(GCC_WARNINGS
       ${CLANG_WARNINGS}
       -Wmisleading-indentation # warn if indentation implies blocks where blocks
@@ -75,8 +71,15 @@ function(set_project_warnings project_name)
       -Wlogical-op   # warn about logical operations being used where bitwise were
                      # probably wanted
       -Wuseless-cast # warn if you perform a cast to the same type
+      -Wno-ignored-attributes
   )
 
+  if (${PROJECT_NAME}_WARNINGS_AS_ERRORS)
+    set(CLANG_WARNINGS ${CLANG_WARNINGS} -Werror)
+    set(GCC_WARNINGS ${GCC_WARNINGS} -Werror)
+    set(MSVC_WARNINGS ${MSVC_WARNINGS} /WX)
+  endif()
+
   if(MSVC)
     set(PROJECT_WARNINGS ${MSVC_WARNINGS})
   elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
@@ -87,7 +90,7 @@ function(set_project_warnings project_name)
     message(AUTHOR_WARNING "No compiler warnings set for '${CMAKE_CXX_COMPILER_ID}' compiler.")
   endif()
 
-  target_compile_options(${project_name} PUBLIC ${PROJECT_WARNINGS})
+  add_compile_options(${PROJECT_WARNINGS})
 
   if(NOT TARGET ${project_name})
     message(AUTHOR_WARNING "${project_name} is not a target, thus no compiler warning were added.")

diff --git a/cmake/Sanitizers.cmake b/cmake/Sanitizers.cmake
@@ -0,0 +1,16 @@
+function(enable_sanitizer sanitizer)
+  # Add for all targets
+  add_compile_options(-fsanitize=${sanitizer})
+  add_link_options(-fsanitize=${sanitizer})
+  message(STATUS "Enabled ${sanitizer} sanitizer.")
+endfunction()
+
+if(${PROJECT_NAME_UPPERCASE}_ENABLE_ASAN)
+  enable_sanitizer("address")
+endif()
+if(${PROJECT_NAME_UPPERCASE}_ENABLE_TSAN)
+  enable_sanitizer("thread")
+endif()
+if(${PROJECT_NAME_UPPERCASE}_ENABLE_UBSAN)
+  enable_sanitizer("undefined")
+endif()