Skip to content

Commit

Permalink
dynamic store arg patch
Browse files Browse the repository at this point in the history
  • Loading branch information
@odygrd
odygrd committed May 17, 2024
1 parent 877adc4 commit 19a91c4
Show file tree
Hide file tree
Showing 6 changed files with 74 additions and 113 deletions.
10 changes: 4 additions & 6 deletions quill/include/quill/Logger.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,10 @@ class LoggerImpl : public detail::LoggerBase
detail::ThreadContext* const thread_context =
quill::detail::get_local_thread_context<frontend_options_t>();

constexpr uint32_t c_string_count =
(0u + ... + detail::CStyleStringCount<detail::remove_cvref_t<Args>>::value);
size_t c_string_sizes[(c_string_count > 1 ? c_string_count : 1)];

// Need to reserve additional space as we will be aligning the pointer
size_t total_size = sizeof(current_timestamp) + (sizeof(uintptr_t) * 3) +
detail::calculate_args_size_and_populate_string_lengths(c_string_sizes, fmt_args...);
detail::calculate_args_size_and_populate_string_lengths(
thread_context->get_c_style_string_length_cache(), fmt_args...);

if (dynamic_log_level != LogLevel::None)
{
Expand Down Expand Up @@ -188,7 +185,7 @@ class LoggerImpl : public detail::LoggerBase
write_buffer += sizeof(uintptr_t);

// encode remaining arguments
detail::encode(write_buffer, c_string_sizes, fmt_args...);
detail::encode(write_buffer, thread_context->get_c_style_string_length_cache(), fmt_args...);

if (dynamic_log_level != LogLevel::None)
{
Expand All @@ -205,6 +202,7 @@ class LoggerImpl : public detail::LoggerBase

thread_context->get_spsc_queue<frontend_options_t::queue_type>().finish_write(static_cast<uint32_t>(total_size));
thread_context->get_spsc_queue<frontend_options_t::queue_type>().commit_write();
thread_context->get_c_style_string_length_cache().clear();

return true;
}
Expand Down
10 changes: 7 additions & 3 deletions quill/include/quill/core/BoundedSPSCQueue.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ class BoundedSPSCQueueImpl
_mask(_capacity - 1),
_bytes_per_batch(static_cast<integer_type>(_capacity * static_cast<double>(reader_store_percent) / 100.0)),
_storage(static_cast<std::byte*>(_alloc_aligned(2ull * static_cast<uint64_t>(_capacity),
CACHE_LINE_ALIGNED, huges_pages_enabled)))
CACHE_LINE_ALIGNED, huges_pages_enabled))),
_huge_pages_enabled(huges_pages_enabled)
{
std::memset(_storage, 0, 2ull * static_cast<uint64_t>(_capacity));

Expand Down Expand Up @@ -189,6 +190,8 @@ class BoundedSPSCQueueImpl
return static_cast<integer_type>(_capacity);
}

QUILL_NODISCARD bool huge_pages_enabled() const noexcept { return _huge_pages_enabled; }

private:
#if defined(QUILL_X86ARCH)
QUILL_ALWAYS_INLINE_HOT void _flush_cachelines(integer_type& last, integer_type offset)
Expand Down Expand Up @@ -221,13 +224,13 @@ class BoundedSPSCQueueImpl
* Aligned alloc
* @param size number of bytes to allocate. An integral multiple of alignment
* @param alignment specifies the alignment. Must be a valid alignment supported by the implementation.
* @param huges_pages_enabled allocate huge pages, only suported on linux
* @param huges_pages_enabled allocate huge pages, only supported on linux
* @return On success, returns the pointer to the beginning of newly allocated memory.
* To avoid a memory leak, the returned pointer must be deallocated with _free_aligned().
* @throws std::system_error on failure
*/

QUILL_NODISCARD static void* _alloc_aligned(size_t size, size_t alignment, bool huges_pages_enabled = false)
QUILL_NODISCARD static void* _alloc_aligned(size_t size, size_t alignment, bool huges_pages_enabled)
{
#if defined(_WIN32)
void* p = _aligned_malloc(size, alignment);
Expand Down Expand Up @@ -306,6 +309,7 @@ class BoundedSPSCQueueImpl
integer_type const _mask;
integer_type const _bytes_per_batch;
std::byte* const _storage{nullptr};
bool _huge_pages_enabled;

alignas(CACHE_LINE_ALIGNED) std::atomic<integer_type> _atomic_writer_pos{0};
alignas(CACHE_LINE_ALIGNED) integer_type _writer_pos{0};
Expand Down
127 changes: 44 additions & 83 deletions quill/include/quill/core/Codec.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,36 +60,6 @@ constexpr auto strnlen =
#endif
;

/** Class template to check if a type is a c style string **/
template <typename Arg, typename = void>
struct CStyleStringCount
{
// Disable the template to have control over which types that contain c style strings are passed
static_assert(always_false_v<Arg>, "Unsupported Type");
};

/***/
template <typename Arg>
struct CStyleStringCount<
Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, std::string>, std::is_same<Arg, std::string_view>, std::is_arithmetic<Arg>, std::is_enum<Arg>>>>
{
static constexpr uint32_t value = 0;
};

/***/
template <typename Arg>
struct CStyleStringCount<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, char*>, std::is_same<Arg, char const*>>>>
{
static constexpr uint32_t value = 1;
};

/***/
template <std::size_t N>
struct CStyleStringCount<char[N]>
{
static constexpr uint32_t value = 1;
};

#if defined(_WIN32)
/**
* Return the size required to encode a wide string
Expand All @@ -110,21 +80,13 @@ void inline wide_string_to_narrow(void* dest, size_t required_bytes, std::wstrin
WideCharToMultiByte(CP_UTF8, 0, s.data(), static_cast<int>(s.size()), static_cast<char*>(dest),
static_cast<int>(required_bytes), nullptr, nullptr);
}

/***/
template <typename T>
struct CStyleStringCount<
T, std::enable_if_t<std::disjunction_v<std::is_same<T, wchar_t*>, std::is_same<T, wchar_t const*>, std::is_same<T, std::wstring>, std::is_same<T, std::wstring_view>>>>
{
static constexpr uint32_t value = 1;
};
#endif

/** typename = void for specializations with enable_if **/
template <typename Arg, typename = void>
struct CalculateArgSize
{
QUILL_ATTRIBUTE_HOT static inline size_t calculate(size_t*, uint32_t&, Arg const&) noexcept
QUILL_ATTRIBUTE_HOT static inline size_t calculate(std::vector<size_t>&, Arg const&) noexcept
{
static_assert(always_false_v<Arg>, "Unsupported type");
return 0;
Expand All @@ -135,7 +97,7 @@ struct CalculateArgSize
template <typename Arg>
struct CalculateArgSize<Arg, std::enable_if_t<std::disjunction_v<std::is_arithmetic<Arg>, std::is_enum<Arg>>>>
{
QUILL_ATTRIBUTE_HOT static constexpr size_t calculate(size_t*, uint32_t&, Arg) noexcept
QUILL_ATTRIBUTE_HOT static constexpr size_t calculate(std::vector<size_t>&, Arg) noexcept
{
return static_cast<size_t>(sizeof(Arg));
}
Expand All @@ -145,34 +107,32 @@ struct CalculateArgSize<Arg, std::enable_if_t<std::disjunction_v<std::is_arithme
template <typename Arg>
struct CalculateArgSize<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, char*>, std::is_same<Arg, char const*>>>>
{
QUILL_ATTRIBUTE_HOT static inline size_t calculate(size_t* c_style_string_lengths,
uint32_t& c_style_string_lengths_index,
QUILL_ATTRIBUTE_HOT static inline size_t calculate(std::vector<size_t>& c_style_string_length_cache,
char const* arg) noexcept
{
// include one extra for the zero termination
c_style_string_lengths[c_style_string_lengths_index] = static_cast<size_t>(strlen(arg) + 1u);
return c_style_string_lengths[c_style_string_lengths_index++];
c_style_string_length_cache.push_back(static_cast<size_t>(strlen(arg) + 1u));
return c_style_string_length_cache.back();
}
};

/***/
template <std::size_t N>
struct CalculateArgSize<char[N]>
{
QUILL_ATTRIBUTE_HOT static inline size_t calculate(size_t* c_style_string_lengths,
uint32_t& c_style_string_lengths_index,
QUILL_ATTRIBUTE_HOT static inline size_t calculate(std::vector<size_t>& c_style_string_length_cache,
char const (&arg)[N]) noexcept
{
c_style_string_lengths[c_style_string_lengths_index] = static_cast<size_t>(strnlen(arg, N) + 1u);
return c_style_string_lengths[c_style_string_lengths_index++];
c_style_string_length_cache.push_back(static_cast<size_t>(strnlen(arg, N) + 1u));
return c_style_string_length_cache.back();
}
};

/***/
template <typename Arg>
struct CalculateArgSize<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, std::string>, std::is_same<Arg, std::string_view>>>>
{
QUILL_ATTRIBUTE_HOT static inline size_t calculate(size_t*, uint32_t&, Arg const& arg) noexcept
QUILL_ATTRIBUTE_HOT static inline size_t calculate(std::vector<size_t>&, Arg const& arg) noexcept
{
// for std::string we also need to store the size in order to correctly retrieve it
// the reason for this is that if we create e.g:
Expand All @@ -187,27 +147,26 @@ struct CalculateArgSize<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Ar
template <typename Arg>
struct CalculateArgSize<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, wchar_t*>, std::is_same<Arg, wchar_t const*>>>>
{
QUILL_ATTRIBUTE_HOT static inline size_t calculate(size_t* c_style_string_lengths,
uint32_t& c_style_string_lengths_index,
QUILL_ATTRIBUTE_HOT static inline size_t calculate(std::vector<size_t>& c_style_string_length_cache,
wchar_t const* arg) noexcept
{
// Those strings won't be zero terminated after encoding, we will zero terminate them ourselves
// so we add + 1 to the size
c_style_string_lengths[c_style_string_lengths_index] =
get_wide_string_encoding_size(std::wstring_view{arg, wcslen(arg)}) + 1;
return static_cast<size_t>(c_style_string_lengths[c_style_string_lengths_index++]);
c_style_string_length_cache.push_back(
get_wide_string_encoding_size(std::wstring_view{arg, wcslen(arg)}) + 1u);
return static_cast<size_t>(get_wide_string_encoding_size.back());
}
};

/***/
template <typename Arg>
struct CalculateArgSize<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, std::wstring>, std::is_same<Arg, std::wstring_view>>>>
{
QUILL_ATTRIBUTE_HOT static inline size_t calculate(size_t* c_style_string_lengths,
uint32_t& c_style_string_lengths_index, Arg const& arg) noexcept
QUILL_ATTRIBUTE_HOT static inline size_t calculate(std::vector<size_t>& c_style_string_length_cache,
Arg const& arg) noexcept
{
c_style_string_lengths[c_style_string_lengths_index] = get_wide_string_encoding_size(arg);
return static_cast<size_t>(sizeof(size_t) + c_style_string_lengths[c_style_string_lengths_index++]);
c_style_string_length_cache.push_back(get_wide_string_encoding_size(arg));
return static_cast<size_t>(sizeof(size_t) + c_style_string_length_cache.back());
}
};
#endif
Expand All @@ -221,19 +180,16 @@ struct CalculateArgSize<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Ar
*/
template <typename... Args>
QUILL_NODISCARD QUILL_ATTRIBUTE_HOT inline size_t calculate_args_size_and_populate_string_lengths(
QUILL_MAYBE_UNUSED size_t* c_style_string_lengths, Args const&... args) noexcept
QUILL_MAYBE_UNUSED std::vector<size_t>& c_style_string_length_cache, Args const&... args) noexcept
{
QUILL_MAYBE_UNUSED uint32_t c_style_string_lengths_index{0};
return (0u + ... +
CalculateArgSize<remove_cvref_t<Args>>::calculate(c_style_string_lengths,
c_style_string_lengths_index, args));
return (0u + ... + CalculateArgSize<remove_cvref_t<Args>>::calculate(c_style_string_length_cache, args));
}

/** typename = void for specializations with enable_if **/
template <typename Arg, typename = void>
struct Encode
{
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*&, size_t const*, uint32_t&, Arg const&) noexcept
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*&, std::vector<size_t> const&, uint32_t&, Arg const&) noexcept
{
static_assert(always_false_v<Arg>, "Unsupported type");
}
Expand All @@ -243,7 +199,8 @@ struct Encode
template <typename Arg>
struct Encode<Arg, std::enable_if_t<std::disjunction_v<std::is_arithmetic<Arg>, std::is_enum<Arg>>>>
{
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, size_t const*, uint32_t&, Arg arg) noexcept
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, std::vector<size_t> const&,
uint32_t&, Arg arg) noexcept
{
std::memcpy(buffer, &arg, sizeof(arg));
buffer += sizeof(arg);
Expand All @@ -254,11 +211,12 @@ struct Encode<Arg, std::enable_if_t<std::disjunction_v<std::is_arithmetic<Arg>,
template <typename Arg>
struct Encode<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, char*>, std::is_same<Arg, char const*>>>>
{
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, size_t const* c_style_string_lengths,
uint32_t& c_style_string_lengths_index, char const* arg) noexcept
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, std::vector<size_t> const& c_style_string_lengths_cache,
uint32_t& c_style_string_lengths_cache_index,
char const* arg) noexcept
{
// null terminator is included in the len for c style strings
size_t const len = c_style_string_lengths[c_style_string_lengths_index++];
size_t const len = c_style_string_lengths_cache[c_style_string_lengths_cache_index++];
std::memcpy(buffer, arg, len);
buffer += len;
}
Expand All @@ -268,7 +226,8 @@ struct Encode<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, char*>,
template <typename Arg>
struct Encode<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, std::string>, std::is_same<Arg, std::string_view>>>>
{
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, size_t const*, uint32_t&, Arg const& arg) noexcept
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, std::vector<size_t> const&,
uint32_t&, Arg const& arg) noexcept
{
// for std::string we store the size first, in order to correctly retrieve it
// Copy the length first and then the actual string
Expand All @@ -289,11 +248,11 @@ struct Encode<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, std::st
template <std::size_t N>
struct Encode<char[N]>
{
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, size_t const* c_style_string_lengths,
uint32_t& c_style_string_lengths_index,
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, std::vector<size_t> const& c_style_string_lengths_cache,
uint32_t& c_style_string_lengths_cache_index,
char const (&arg)[N]) noexcept
{
size_t const len = c_style_string_lengths[c_style_string_lengths_index++];
size_t const len = c_style_string_lengths_cache[c_style_string_lengths_cache_index++];

if (QUILL_UNLIKELY(len > N))
{
Expand All @@ -316,10 +275,11 @@ struct Encode<char[N]>
template <typename Arg>
struct Encode<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, wchar_t*>, std::is_same<Arg, wchar_t const*>>>>
{
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, size_t const* c_style_string_lengths,
uint32_t& c_style_string_lengths_index, wchar_t const* arg) noexcept
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, std::vector<size_t> const& c_style_string_lengths_cache,
uint32_t& c_style_string_lengths_cache_index,
wchar_t const* arg) noexcept
{
size_t const total_len = c_style_string_lengths[c_style_string_lengths_index++];
size_t const total_len = c_style_string_lengths_cache[c_style_string_lengths_cache_index++];
size_t const str_len = total_len - 1; // excluding the zero termination

if (str_len != 0)
Expand All @@ -338,11 +298,12 @@ struct Encode<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, wchar_t
template <typename Arg>
struct Encode<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, std::wstring>, std::is_same<Arg, std::wstring_view>>>>
{
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, size_t const* c_style_string_lengths,
uint32_t& c_style_string_lengths_index, Arg const& arg) noexcept
QUILL_ATTRIBUTE_HOT static inline void encode(std::byte*& buffer, std::vector<size_t> const& c_style_string_lengths_cache,
uint32_t& c_style_string_lengths_cache_index,
Arg const& arg) noexcept
{
// for std::wstring we store the size first, in order to correctly retrieve it
size_t const len = c_style_string_lengths[c_style_string_lengths_index++];
size_t const len = c_style_string_lengths_cache[c_style_string_lengths_cache_index++];
std::memcpy(buffer, &len, sizeof(len));

if (len != 0)
Expand All @@ -362,11 +323,11 @@ struct Encode<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, std::ws
* @param args The arguments to be encoded.
*/
template <typename... Args>
QUILL_ATTRIBUTE_HOT inline void encode(std::byte*& buffer, QUILL_MAYBE_UNUSED size_t const* c_style_string_lengths,
QUILL_ATTRIBUTE_HOT inline void encode(std::byte*& buffer, std::vector<size_t> const& c_style_string_length_cache,
Args const&... args) noexcept
{
QUILL_MAYBE_UNUSED uint32_t c_style_string_lengths_index{0};
(Encode<Args>::encode(buffer, c_style_string_lengths, c_style_string_lengths_index, args), ...);
QUILL_MAYBE_UNUSED uint32_t c_style_string_lengths_cache_index{0};
(Encode<Args>::encode(buffer, c_style_string_length_cache, c_style_string_lengths_cache_index, args), ...);
}

/** typename = void for specializations with enable_if **/
Expand Down Expand Up @@ -510,8 +471,8 @@ struct Decode<Arg, std::enable_if_t<std::disjunction_v<std::is_same<Arg, std::ws
#endif

template <typename... Args>
QUILL_ATTRIBUTE_HOT inline void decode(std::byte*& buffer,
fmtquill::dynamic_format_arg_store<fmtquill::format_context>* args_store) noexcept
QUILL_ATTRIBUTE_HOT inline void decode(
std::byte*& buffer, QUILL_MAYBE_UNUSED fmtquill::dynamic_format_arg_store<fmtquill::format_context>* args_store) noexcept
{
(Decode<Args>::decode(buffer, args_store), ...);
}
Expand Down
9 changes: 9 additions & 0 deletions quill/include/quill/core/ThreadContextManager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ class ThreadContext
new (&_spsc_queue_union.bounded_spsc_queue)
BoundedSPSCQueue{initial_spsc_queue_capacity, huges_pages_enabled};
}

_c_style_string_length_cache.reserve(8);
}

/***/
Expand Down Expand Up @@ -109,6 +111,12 @@ class ThreadContext
}
}

/***/
QUILL_NODISCARD_ALWAYS_INLINE_HOT std::vector<size_t>& get_c_style_string_length_cache() noexcept
{
return _c_style_string_length_cache;
}

/***/
QUILL_NODISCARD_ALWAYS_INLINE_HOT bool has_bounded_queue_type() const noexcept
{
Expand Down Expand Up @@ -181,6 +189,7 @@ class ThreadContext
friend class detail::BackendWorker;

SpscQueueUnion _spsc_queue_union; /** queue for this thread */
std::vector<size_t> _c_style_string_length_cache; /** cache used for the length of c-style string log arguments **/
std::string _thread_id = std::to_string(get_thread_id()); /**< cache this thread pid */
std::string _thread_name = get_thread_name(); /**< cache this thread name */
std::shared_ptr<UnboundedTransitEventBuffer> _transit_event_buffer; /** backend thread buffer. this could be unique_ptr but it is shared_ptr because of the forward declaration */
Expand Down
Loading

0 comments on commit 19a91c4

Please sign in to comment.