support calculate blob info of valid blob and valid entry, this entry will benifit from value handle bytedance#199

add tickers of read blob valid bytedance#199

fix invalid_file_cnt statistic

fix invalid_file_cnt_ statistic bytedance#199

refine code style bytedance#199

Author: yapple

db/compaction_job.cc

@@ -1490,6 +1490,14 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
   stream.EndArray();
   stream << "blob_count";
   stream << vstorage->NumLevelFiles(-1);
+  stream << "invalid_file_cnt";
+  stream << vstorage->invalid_file_cnt();
+  stream << "valid_file_cnt";
+  stream << vstorage->valid_file_cnt();
+  stream << "invalid_entry_cnt";
+  stream << vstorage->invalid_entry_cnt();
+  stream << "valid_entry_cnt";
+  stream << vstorage->valid_entry_cnt();
 
   CleanupCompaction();
   return status;

db/flush_job.cc

@@ -283,6 +283,14 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker) {
   stream.EndArray();
   stream << "blob_count";
   stream << vstorage->NumLevelFiles(-1);
+  stream << "invalid_file_cnt";
+  stream << vstorage->invalid_file_cnt();
+  stream << "valid_file_cnt";
+  stream << vstorage->valid_file_cnt();
+  stream << "invalid_entry_cnt";
+  stream << vstorage->invalid_entry_cnt();
+  stream << "valid_entry_cnt";
+  stream << vstorage->valid_entry_cnt();
 
   stream << "immutable_memtables" << cfd_->imm()->NumNotFlushed();
 

db/version_builder.cc

@@ -735,6 +735,7 @@ class VersionBuilder::Rep {
     vstorage->ResetVersionBuilderContext(context_.release());
     vstorage->ComputeBlobOverlapScore();
     vstorage->CalculateEdge();
+    vstorage->CalculateBlobInfo();
   }
 
   void LoadTableHandlers(InternalStats* internal_stats,

db/version_set.cc

@@ -1216,6 +1216,11 @@ VersionStorageInfo::VersionStorageInfo(
       file_indexer_(user_comparator),
       compaction_style_(compaction_style),
       files_(new std::vector<FileMetaData*>[num_levels_ + 1]),
+      edge_cnt_levels_(num_levels_),
+      valid_file_cnt_(0),
+      valid_entry_cnt_(0),
+      invalid_file_cnt_(0),
+      invalid_entry_cnt_(0),
       base_level_(num_levels_ == 1 ? -1 : 1),
       level_multiplier_(0.0),
       files_by_compaction_pri_(num_levels_),
@@ -1275,6 +1280,11 @@ Status Version::fetch_buffer(LazyBuffer* buffer) const {
                  context->data[1]);
   uint64_t sequence = context->data[2];
   auto pair = *reinterpret_cast<DependenceMap::value_type*>(context->data[3]);
+  if (pair.second->fd.GetNumber() != pair.first) {
+    RecordTick(db_statistics_, READ_BLOB_INVALID);
+  } else {
+    RecordTick(db_statistics_, READ_BLOB_VALID);
+  }
   bool value_found = false;
   SequenceNumber context_seq;
   GetContext get_context(cfd_->internal_comparator().user_comparator(), nullptr,
@@ -4864,4 +4874,31 @@ void VersionStorageInfo::CalculateEdge() {
   }
 }
 
+void VersionStorageInfo::CalculateBlobInfo() {
+  valid_file_cnt_ = 0;
+  valid_entry_cnt_ = 0;
+  invalid_file_cnt_ = 0;
+  invalid_entry_cnt_ = 0;
+  chash_map<uint64_t, uint64_t> file_map;
+  chash_set<uint64_t> invalid_file_set;
+  for (int i = 0; i < num_levels_; i++) {
+    for (auto& f : LevelFiles(i)) {
+      for (auto fn : f->prop.dependence) {
+        file_map[fn.file_number] += fn.entry_count;
+      }
+    }
+  }
+  for (auto f : file_map) {
+    uint64_t newest_file_number =
+        dependence_map_.find(f.first)->second->fd.GetNumber();
+    if (newest_file_number != f.first) {
+      invalid_file_set.insert(newest_file_number);
+      invalid_entry_cnt_ += f.second;
+    } else {
+      valid_file_cnt_++;
+      valid_entry_cnt_ += f.second;
+    }
+  }
+  invalid_file_cnt_ = invalid_file_set.size();
+}
 }  // namespace TERARKDB_NAMESPACE

db/version_set.h

@@ -503,6 +503,12 @@ class VersionStorageInfo {
   void CalculateEdge();
   std::vector<uint64_t> edge_cnt_levels() const { return edge_cnt_levels_; }
 
+  void CalculateBlobInfo();
+  uint64_t valid_file_cnt() const { return valid_file_cnt_; }
+  uint64_t valid_entry_cnt() const { return valid_entry_cnt_; }
+  uint64_t invalid_file_cnt() const { return invalid_file_cnt_; }
+  uint64_t invalid_entry_cnt() const { return invalid_entry_cnt_; }
+
  private:
   const InternalKeyComparator* internal_comparator_;
   const Comparator* user_comparator_;
@@ -527,6 +533,11 @@ class VersionStorageInfo {
   // We Change it when new version is build
   std::vector<uint64_t> edge_cnt_levels_;
 
+  uint64_t valid_file_cnt_;
+  uint64_t valid_entry_cnt_;
+  uint64_t invalid_file_cnt_;
+  uint64_t invalid_entry_cnt_;
+
   // Dependence files both in files[-1] and dependence_map
   DependenceMap dependence_map_;
 

include/rocksdb/statistics.h

@@ -258,6 +258,10 @@ enum Tickers : uint32_t {
   GC_TOUCH_FILES,
   GC_SKIP_GET_BY_SEQ,
   GC_SKIP_GET_BY_FILE,
+
+  READ_BLOB_VALID,
+  READ_BLOB_INVALID,
+
   TICKER_ENUM_MAX
 };
 

java/rocksjni/portal.h

@@ -3476,9 +3476,12 @@ class TickerTypeJni {
         return 0x63;
       case TERARKDB_NAMESPACE::Tickers::GC_SKIP_GET_BY_FILE:
         return 0x64;
-      case TERARKDB_NAMESPACE::Tickers::TICKER_ENUM_MAX:
+      case TERARKDB_NAMESPACE::Tickers::READ_BLOB_VALID:
         return 0x65;
-
+      case TERARKDB_NAMESPACE::Tickers::READ_BLOB_INVALID:
+        return 0x66;
+      case TERARKDB_NAMESPACE::Tickers::TICKER_ENUM_MAX:
+        return 0x67;
       default:
         // undefined/default
         return 0x0;

monitoring/statistics.cc

@@ -139,6 +139,8 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
     {GC_TOUCH_FILES, "rocksdb.num.gc.touch_files"},
     {GC_SKIP_GET_BY_SEQ, "rocksdb.num.gc.skip_by_seqno"},
     {GC_SKIP_GET_BY_FILE, "rocksdb.num.gc.skip_by_file_meta"},
+    {READ_BLOB_VALID, "rocksdb.num.read.blob_valid"},
+    {READ_BLOB_INVALID, "rocksdb.num.read.blob_invalid"},
 };
 
 const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {