-
Notifications
You must be signed in to change notification settings - Fork 41
Support ListOffset/SubscribeBatch/DropTable for cpp bindings #100
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -63,6 +63,24 @@ enum class DatumType { | |
| Bytes = 7, | ||
| }; | ||
|
|
||
| constexpr int64_t EARLIEST_OFFSET = -2; | ||
| constexpr int64_t LATEST_OFFSET = -1; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm curious about
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sometimes, lagged consumption can impact business operations. In such cases, the business may choose to resolve the issue by reverting to the latest consumption. |
||
|
|
||
| enum class OffsetSpec { | ||
| Earliest = 0, | ||
| Latest = 1, | ||
| Timestamp = 2, | ||
| }; | ||
|
|
||
| struct OffsetQuery { | ||
| OffsetSpec spec; | ||
| int64_t timestamp{0}; | ||
|
|
||
| static OffsetQuery Earliest() { return {OffsetSpec::Earliest, 0}; } | ||
| static OffsetQuery Latest() { return {OffsetSpec::Latest, 0}; } | ||
| static OffsetQuery FromTimestamp(int64_t ts) { return {OffsetSpec::Timestamp, ts}; } | ||
| }; | ||
|
|
||
| struct Result { | ||
| int32_t error_code{0}; | ||
| std::string error_message; | ||
|
|
@@ -301,6 +319,7 @@ struct GenericRow { | |
| }; | ||
|
|
||
| struct ScanRecord { | ||
| int32_t bucket_id; | ||
| int64_t offset; | ||
| int64_t timestamp; | ||
| GenericRow row; | ||
|
|
@@ -324,6 +343,11 @@ struct BucketOffset { | |
| int64_t offset; | ||
| }; | ||
|
|
||
| struct BucketSubscription { | ||
| int32_t bucket_id; | ||
| int64_t offset; | ||
| }; | ||
|
|
||
| struct LakeSnapshot { | ||
| int64_t snapshot_id; | ||
| std::vector<BucketOffset> bucket_offsets; | ||
|
|
@@ -372,10 +396,17 @@ class Admin { | |
| const TableDescriptor& descriptor, | ||
| bool ignore_if_exists = false); | ||
|
|
||
| Result DropTable(const TablePath& table_path, bool ignore_if_not_exists = false); | ||
|
|
||
| Result GetTable(const TablePath& table_path, TableInfo& out); | ||
|
|
||
| Result GetLatestLakeSnapshot(const TablePath& table_path, LakeSnapshot& out); | ||
|
|
||
| Result ListOffsets(const TablePath& table_path, | ||
| const std::vector<int32_t>& bucket_ids, | ||
| const OffsetQuery& offset_query, | ||
| std::unordered_map<int32_t, int64_t>& out); | ||
|
|
||
| private: | ||
| friend class Connection; | ||
| Admin(ffi::Admin* admin) noexcept; | ||
|
|
@@ -448,6 +479,7 @@ class LogScanner { | |
| bool Available() const; | ||
|
|
||
| Result Subscribe(int32_t bucket_id, int64_t start_offset); | ||
| Result Subscribe(const std::vector<BucketSubscription>& bucket_offsets); | ||
| Result Poll(int64_t timeout_ms, ScanRecords& out); | ||
|
|
||
| private: | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -66,6 +66,16 @@ Result Admin::CreateTable(const TablePath& table_path, | |
| return utils::from_ffi_result(ffi_result); | ||
| } | ||
|
|
||
| Result Admin::DropTable(const TablePath& table_path, bool ignore_if_not_exists) { | ||
| if (!Available()) { | ||
| return utils::make_error(1, "Admin not available"); | ||
| } | ||
|
|
||
| auto ffi_path = utils::to_ffi_table_path(table_path); | ||
| auto ffi_result = admin_->drop_table(ffi_path, ignore_if_not_exists); | ||
| return utils::from_ffi_result(ffi_result); | ||
| } | ||
|
|
||
| Result Admin::GetTable(const TablePath& table_path, TableInfo& out) { | ||
| if (!Available()) { | ||
| return utils::make_error(1, "Admin not available"); | ||
|
|
@@ -98,4 +108,36 @@ Result Admin::GetLatestLakeSnapshot(const TablePath& table_path, LakeSnapshot& o | |
| return result; | ||
| } | ||
|
|
||
| Result Admin::ListOffsets(const TablePath& table_path, | ||
| const std::vector<int32_t>& bucket_ids, | ||
| const OffsetQuery& offset_query, | ||
| std::unordered_map<int32_t, int64_t>& out) { | ||
| if (!Available()) { | ||
| return utils::make_error(1, "Admin not available"); | ||
| } | ||
|
|
||
| auto ffi_path = utils::to_ffi_table_path(table_path); | ||
|
|
||
|
||
| rust::Vec<int32_t> rust_bucket_ids; | ||
| for (int32_t id : bucket_ids) { | ||
| rust_bucket_ids.push_back(id); | ||
| } | ||
|
|
||
| ffi::FfiOffsetQuery ffi_query; | ||
| ffi_query.offset_type = static_cast<int32_t>(offset_query.spec); | ||
| ffi_query.timestamp = offset_query.timestamp; | ||
|
|
||
| auto ffi_result = admin_->list_offsets(ffi_path, std::move(rust_bucket_ids), ffi_query); | ||
|
|
||
|
||
| auto result = utils::from_ffi_result(ffi_result.result); | ||
| if (result.Ok()) { | ||
| out.clear(); | ||
| for (const auto& pair : ffi_result.bucket_offsets) { | ||
| out[pair.bucket_id] = pair.offset; | ||
| } | ||
| } | ||
|
|
||
| return result; | ||
| } | ||
|
|
||
| } // namespace fluss | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The constants EARLIEST_OFFSET and LATEST_OFFSET are defined but not used in the codebase shown. Consider documenting their intended use or removing them if they are not needed. These constants could be useful for users of the API to pass to Subscribe methods, but their purpose should be documented.