Merge branch 'main' into retain-float-format-docs

Author: gibber9809

components/core/CMakeLists.txt

@@ -357,6 +357,8 @@ set(SOURCE_FILES_clp_s_unitTest
     src/clp_s/FileReader.hpp
     src/clp_s/FileWriter.cpp
     src/clp_s/FileWriter.hpp
+    src/clp_s/FloatFormatEncoding.cpp
+    src/clp_s/FloatFormatEncoding.hpp
     src/clp_s/InputConfig.cpp
     src/clp_s/InputConfig.hpp
     src/clp_s/JsonConstructor.cpp
@@ -700,6 +702,7 @@ set(SOURCE_FILES_unitTest
         tests/test-ffi_KeyValuePairLogEvent.cpp
         tests/test-ffi_SchemaTree.cpp
         tests/test-FileDescriptorReader.cpp
+        tests/test-FloatFormatEncoding.cpp
         tests/test-GlobalMetadataDBConfig.cpp
         tests/test-GrepCore.cpp
         tests/test-hash_utils.cpp

components/core/cmake/Options/options.cmake

@@ -213,6 +213,7 @@ function(set_clp_s_archivereader_dependencies)
         CLP_NEED_MSGPACKCXX
         CLP_NEED_NLOHMANN_JSON
         CLP_NEED_SPDLOG
+        CLP_NEED_YSTDLIB
     )
 endfunction()
 
@@ -234,6 +235,7 @@ function(set_clp_s_archivewriter_dependencies)
         CLP_NEED_NLOHMANN_JSON
         CLP_NEED_SIMDJSON
         CLP_NEED_SPDLOG
+        CLP_NEED_YSTDLIB
     )
 endfunction()
 

components/core/src/clp_s/ArchiveReader.cpp

@@ -194,6 +194,12 @@ BaseColumnReader* ArchiveReader::append_reader_column(SchemaReader& reader, int3
         case NodeType::Float:
             column_reader = new FloatColumnReader(column_id);
             break;
+        case NodeType::FormattedFloat:
+            column_reader = new FormattedFloatColumnReader(column_id);
+            break;
+        case NodeType::DictionaryFloat:
+            column_reader = new DictionaryFloatColumnReader(column_id, m_var_dict);
+            break;
         case NodeType::ClpString:
             column_reader = new ClpStringColumnReader(column_id, m_var_dict, m_log_dict);
             break;
@@ -247,6 +253,12 @@ void ArchiveReader::append_unordered_reader_columns(
             case NodeType::Float:
                 column_reader = new FloatColumnReader(column_id);
                 break;
+            case NodeType::FormattedFloat:
+                column_reader = new FormattedFloatColumnReader(column_id);
+                break;
+            case NodeType::DictionaryFloat:
+                column_reader = new DictionaryFloatColumnReader(column_id, m_var_dict);
+                break;
             case NodeType::ClpString:
                 column_reader = new ClpStringColumnReader(column_id, m_var_dict, m_log_dict);
                 break;

components/core/src/clp_s/ArchiveWriter.cpp

@@ -316,6 +316,12 @@ void ArchiveWriter::initialize_schema_writer(SchemaWriter* writer, Schema const&
             case NodeType::Float:
                 writer->append_column(new FloatColumnWriter(id));
                 break;
+            case NodeType::FormattedFloat:
+                writer->append_column(new FormattedFloatColumnWriter(id));
+                break;
+            case NodeType::DictionaryFloat:
+                writer->append_column(new DictionaryFloatColumnWriter(id, m_var_dict));
+                break;
             case NodeType::ClpString:
                 writer->append_column(new ClpStringColumnWriter(id, m_var_dict, m_log_dict));
                 break;

components/core/src/clp_s/CMakeLists.txt

@@ -232,6 +232,8 @@ set(
         DictionaryWriter.cpp
         DictionaryWriter.hpp
         ErrorCode.hpp
+        FloatFormatEncoding.cpp
+        FloatFormatEncoding.hpp
         JsonFileIterator.cpp
         JsonFileIterator.hpp
         JsonParser.cpp
@@ -274,6 +276,7 @@ if(CLP_BUILD_CLP_S_ARCHIVEWRITER)
                 msgpack-cxx
                 nlohmann_json::nlohmann_json
                 simdjson::simdjson
+                ystdlib::error_handling
                 PRIVATE
                 Boost::url
                 clp_s::clp_dependencies
@@ -299,6 +302,8 @@ set(
         DictionaryEntry.hpp
         DictionaryReader.hpp
         ErrorCode.hpp
+        FloatFormatEncoding.cpp
+        FloatFormatEncoding.hpp
         JsonSerializer.hpp
         PackedStreamReader.cpp
         PackedStreamReader.hpp
@@ -335,6 +340,7 @@ if(CLP_BUILD_CLP_S_ARCHIVEREADER)
                 clp_s::io
                 msgpack-cxx
                 nlohmann_json::nlohmann_json
+                ystdlib::error_handling
                 PRIVATE
                 Boost::url
                 clp_s::clp_dependencies

components/core/src/clp_s/ColumnReader.cpp

@@ -1,8 +1,14 @@
 #include "ColumnReader.hpp"
 
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <variant>
+
 #include "../clp/EncodedVariableInterpreter.hpp"
 #include "BufferViewReader.hpp"
 #include "ColumnWriter.hpp"
+#include "FloatFormatEncoding.hpp"
 #include "Utils.hpp"
 
 namespace clp_s {
@@ -50,6 +56,11 @@ void FloatColumnReader::load(BufferViewReader& reader, uint64_t num_messages) {
     m_values = reader.read_unaligned_span<double>(num_messages);
 }
 
+void FormattedFloatColumnReader::load(BufferViewReader& reader, uint64_t num_messages) {
+    m_values = reader.read_unaligned_span<double>(num_messages);
+    m_formats = reader.read_unaligned_span<float_format_t>(num_messages);
+}
+
 void
 Int64ColumnReader::extract_string_value_into_buffer(uint64_t cur_message, std::string& buffer) {
     buffer.append(std::to_string(m_values[cur_message]));
@@ -68,6 +79,12 @@ std::variant<int64_t, double, std::string, uint8_t> FloatColumnReader::extract_v
     return m_values[cur_message];
 }
 
+std::variant<int64_t, double, std::string, uint8_t> FormattedFloatColumnReader::extract_value(
+        uint64_t cur_message
+) {
+    return m_values[cur_message];
+}
+
 void BooleanColumnReader::load(BufferViewReader& reader, uint64_t num_messages) {
     m_values = reader.read_unaligned_span<uint8_t>(num_messages);
 }
@@ -77,12 +94,36 @@ FloatColumnReader::extract_string_value_into_buffer(uint64_t cur_message, std::s
     buffer.append(std::to_string(m_values[cur_message]));
 }
 
+void FormattedFloatColumnReader::extract_string_value_into_buffer(
+        uint64_t cur_message,
+        std::string& buffer
+) {
+    buffer.append(restore_encoded_float(m_values[cur_message], m_formats[cur_message]).value());
+}
+
 std::variant<int64_t, double, std::string, uint8_t> BooleanColumnReader::extract_value(
         uint64_t cur_message
 ) {
     return m_values[cur_message];
 }
 
+void DictionaryFloatColumnReader::load(BufferViewReader& reader, uint64_t num_messages) {
+    m_var_dict_ids = reader.read_unaligned_span<variable_dictionary_id_t>(num_messages);
+}
+
+std::variant<int64_t, double, std::string, uint8_t> DictionaryFloatColumnReader::extract_value(
+        uint64_t cur_message
+) {
+    return std::stod(m_var_dict->get_value(m_var_dict_ids[cur_message]));
+}
+
+void DictionaryFloatColumnReader::extract_string_value_into_buffer(
+        uint64_t cur_message,
+        std::string& buffer
+) {
+    buffer.append(m_var_dict->get_value(m_var_dict_ids[cur_message]));
+}
+
 void ClpStringColumnReader::load(BufferViewReader& reader, uint64_t num_messages) {
     m_logtypes = reader.read_unaligned_span<uint64_t>(num_messages);
     size_t encoded_vars_length = reader.read_value<size_t>();

components/core/src/clp_s/ColumnReader.hpp

@@ -1,11 +1,14 @@
 #ifndef CLP_S_COLUMNREADER_HPP
 #define CLP_S_COLUMNREADER_HPP
 
+#include <cstddef>
+#include <cstdint>
 #include <string>
 #include <variant>
 
 #include "BufferViewReader.hpp"
 #include "DictionaryReader.hpp"
+#include "FloatFormatEncoding.hpp"
 #include "SchemaTree.hpp"
 #include "TimestampDictionaryReader.hpp"
 #include "Utils.hpp"
@@ -147,6 +150,66 @@ class FloatColumnReader : public BaseColumnReader {
     UnalignedMemSpan<double> m_values;
 };
 
+class FormattedFloatColumnReader : public BaseColumnReader {
+public:
+    // Constructor
+    explicit FormattedFloatColumnReader(int32_t id) : BaseColumnReader(id) {}
+
+    // Destructor
+    ~FormattedFloatColumnReader() override = default;
+
+    // Methods inherited from BaseColumnReader
+    void load(BufferViewReader& reader, uint64_t num_messages) override;
+
+    NodeType get_type() override { return NodeType::FormattedFloat; }
+
+    std::variant<int64_t, double, std::string, uint8_t> extract_value(
+            uint64_t cur_message
+    ) override;
+
+    /**
+     * Appends the floating point value to the buffer in its original format by decoding the stored
+     * format information.
+     *
+     * @param cur_message
+     * @param buffer
+     */
+    void extract_string_value_into_buffer(uint64_t cur_message, std::string& buffer) override;
+
+private:
+    UnalignedMemSpan<double> m_values;
+    UnalignedMemSpan<float_format_t> m_formats;
+};
+
+class DictionaryFloatColumnReader : public BaseColumnReader {
+public:
+    // Constructor
+    explicit DictionaryFloatColumnReader(
+            int32_t id,
+            std::shared_ptr<VariableDictionaryReader> var_dict
+    )
+            : BaseColumnReader(id),
+              m_var_dict{std::move(var_dict)} {}
+
+    // Destructor
+    ~DictionaryFloatColumnReader() override = default;
+
+    // Methods inherited from BaseColumnReader
+    void load(BufferViewReader& reader, uint64_t num_messages) override;
+
+    NodeType get_type() override { return NodeType::DictionaryFloat; }
+
+    std::variant<int64_t, double, std::string, uint8_t> extract_value(
+            uint64_t cur_message
+    ) override;
+
+    void extract_string_value_into_buffer(uint64_t cur_message, std::string& buffer) override;
+
+private:
+    std::shared_ptr<VariableDictionaryReader> m_var_dict;
+    UnalignedMemSpan<variable_dictionary_id_t> m_var_dict_ids;
+};
+
 class BooleanColumnReader : public BaseColumnReader {
 public:
     // Constructor

components/core/src/clp_s/ColumnWriter.cpp

@@ -1,5 +1,8 @@
 #include "ColumnWriter.hpp"
 
+#include <algorithm>
+#include <cassert>
+#include <cctype>
 #include <cstdint>
 #include <variant>
 
@@ -46,6 +49,33 @@ void FloatColumnWriter::store(ZstdCompressor& compressor) {
     compressor.write(reinterpret_cast<char const*>(m_values.data()), size);
 }
 
+size_t FormattedFloatColumnWriter::add_value(ParsedMessage::variable_t& value) {
+    auto const& [float_value, format]{std::get<std::pair<double, float_format_t>>(value)};
+    m_values.push_back(float_value);
+    m_formats.push_back(format);
+    return sizeof(double) + sizeof(float_format_t);
+}
+
+void FormattedFloatColumnWriter::store(ZstdCompressor& compressor) {
+    assert(m_formats.size() == m_values.size());
+    auto const values_size = m_values.size() * sizeof(double);
+    auto const format_size = m_formats.size() * sizeof(float_format_t);
+    compressor.write(reinterpret_cast<char const*>(m_values.data()), values_size);
+    compressor.write(reinterpret_cast<char const*>(m_formats.data()), format_size);
+}
+
+size_t DictionaryFloatColumnWriter::add_value(ParsedMessage::variable_t& value) {
+    clp::variable_dictionary_id_t id{};
+    m_var_dict->add_entry(std::get<std::string>(value), id);
+    m_var_dict_ids.push_back(id);
+    return sizeof(clp::variable_dictionary_id_t);
+}
+
+void DictionaryFloatColumnWriter::store(ZstdCompressor& compressor) {
+    auto size{m_var_dict_ids.size() * sizeof(clp::variable_dictionary_id_t)};
+    compressor.write(reinterpret_cast<char const*>(m_var_dict_ids.data()), size);
+}
+
 size_t BooleanColumnWriter::add_value(ParsedMessage::variable_t& value) {
     m_values.push_back(std::get<bool>(value) ? 1 : 0);
     return sizeof(uint8_t);

components/core/src/clp_s/ColumnWriter.hpp

@@ -7,6 +7,7 @@
 #include "../clp/Defs.h"
 #include "DictionaryWriter.hpp"
 #include "FileWriter.hpp"
+#include "FloatFormatEncoding.hpp"
 #include "ParsedMessage.hpp"
 #include "TimestampDictionaryWriter.hpp"
 #include "ZstdCompressor.hpp"
@@ -98,6 +99,44 @@ class FloatColumnWriter : public BaseColumnWriter {
     std::vector<double> m_values;
 };
 
+class FormattedFloatColumnWriter : public BaseColumnWriter {
+public:
+    // Constructor
+    explicit FormattedFloatColumnWriter(int32_t id) : BaseColumnWriter(id) {}
+
+    // Destructor
+    ~FormattedFloatColumnWriter() override = default;
+
+    // Methods inherited from BaseColumnWriter
+    size_t add_value(ParsedMessage::variable_t& value) override;
+
+    void store(ZstdCompressor& compressor) override;
+
+private:
+    std::vector<double> m_values;
+    std::vector<float_format_t> m_formats;
+};
+
+class DictionaryFloatColumnWriter : public BaseColumnWriter {
+public:
+    // Constructor
+    DictionaryFloatColumnWriter(int32_t id, std::shared_ptr<VariableDictionaryWriter> var_dict)
+            : BaseColumnWriter(id),
+              m_var_dict(std::move(var_dict)) {}
+
+    // Destructor
+    ~DictionaryFloatColumnWriter() override = default;
+
+    // Methods inherited from BaseColumnWriter
+    size_t add_value(ParsedMessage::variable_t& value) override;
+
+    void store(ZstdCompressor& compressor) override;
+
+private:
+    std::shared_ptr<VariableDictionaryWriter> m_var_dict;
+    std::vector<clp::variable_dictionary_id_t> m_var_dict_ids;
+};
+
 class BooleanColumnWriter : public BaseColumnWriter {
 public:
     // Constructor

components/core/src/clp_s/CommandLineArguments.cpp

@@ -244,6 +244,10 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) {
                     "print-archive-stats",
                     po::bool_switch(&m_print_archive_stats),
                     "Print statistics (json) about the archive after it's compressed."
+            )(
+                    "retain-float-format",
+                    po::bool_switch(&m_retain_float_format),
+                    "Store extra information to losslessly decompress floats."
             )(
                     "single-file-archive",
                     po::bool_switch(&m_single_file_archive),