From 044f8c724bc0c40b37238f7cc9398c334c28792c Mon Sep 17 00:00:00 2001 From: Alex Gaetano Padula Date: Mon, 27 Apr 2026 03:56:00 -0400 Subject: [PATCH] align with tdb 920 with tombstone density capability, and range compaction --- CMakeLists.txt | 2 +- include/tidesdb/tidesdb.hpp | 35 +++++++ src/tidesdb.cpp | 75 ++++++++++++++ tests/tidesdb_test.cpp | 190 ++++++++++++++++++++++++++++++++++++ 4 files changed, 301 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b5f0fa..da51abe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.16) -project(tidesdb_cpp VERSION 2.4.0 LANGUAGES CXX) +project(tidesdb_cpp VERSION 2.5.0 LANGUAGES CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/include/tidesdb/tidesdb.hpp b/include/tidesdb/tidesdb.hpp index 35b3adf..2cd2b1f 100644 --- a/include/tidesdb/tidesdb.hpp +++ b/include/tidesdb/tidesdb.hpp @@ -192,6 +192,10 @@ struct ColumnFamilyConfig std::uint64_t minDiskSpace = 100 * 1024 * 1024; int l1FileCountTrigger = 4; int l0QueueStallThreshold = 20; + double tombstoneDensityTrigger = 0.0; ///< Per-SSTable tombstone density above which + ///< compaction priority escalates (0.0 = disabled) + std::uint64_t tombstoneDensityMinEntries = + 1024; ///< SSTables with fewer entries are ignored by the density trigger bool useBtree = false; ///< Use B+tree format for klog (default: false = block-based) tidesdb_commit_hook_fn commitHookFn = nullptr; ///< Optional commit hook callback (runtime-only) @@ -275,6 +279,8 @@ struct Config float unifiedMemtableSkipListProbability = 0; ///< Skip list probability (0 = default 0.25) SyncMode unifiedMemtableSyncMode = SyncMode::None; ///< Sync mode for unified WAL std::uint64_t unifiedMemtableSyncIntervalUs = 0; ///< Sync interval for unified WAL + int maxConcurrentFlushes = + 0; ///< Global cap on in-flight memtable flushes across all CFs (0 = library default) tidesdb_objstore_t* objectStore = nullptr; ///< Pluggable object store connector (nullptr = local only) std::optional @@ -302,6 +308,12 @@ struct Stats std::uint64_t btreeTotalNodes = 0; ///< Total B+tree nodes across all SSTables std::uint32_t btreeMaxHeight = 0; ///< Maximum tree height across all SSTables double btreeAvgHeight = 0.0; ///< Average tree height across all SSTables + std::uint64_t totalTombstones = 0; ///< Sum of tombstone counts across all SSTables + double tombstoneRatio = 0.0; ///< total_tombstones / total_keys (0.0 if total_keys == 0) + std::vector + levelTombstoneCounts; ///< Per-level tombstone counts (parallels levelKeyCounts) + double maxSstDensity = 0.0; ///< Worst per-SSTable tombstone density observed (0.0 to 1.0) + int maxSstDensityLevel = 0; ///< 1-based level where maxSstDensity was observed (0 if none) }; /** @@ -378,6 +390,29 @@ class ColumnFamily */ void compact(); + /** + * @brief Synchronously compact only SSTables overlapping [startKey, endKey) + * + * Blocks the calling thread until the merge commits or fails. Pass an empty + * optional (std::nullopt) for either endpoint to leave that side unbounded. + * Both endpoints unbounded is rejected with ErrorCode::InvalidArgs - use + * compact() for a full column-family compaction. + * + * @param startKey Start of range (inclusive); std::nullopt = unbounded + * @param endKey End of range (exclusive); std::nullopt = unbounded + */ + void compactRange(const std::optional>& startKey, + const std::optional>& endKey); + + /** + * @brief compactRange overload taking string_view bounds + * + * Pass empty optional for an unbounded endpoint. Empty (but present) + * string_views are forwarded as zero-length keys, matching the C API. + */ + void compactRange(const std::optional& startKey, + const std::optional& endKey); + /** * @brief Manually trigger memtable flush */ diff --git a/src/tidesdb.cpp b/src/tidesdb.cpp index b968bc4..b22f17c 100644 --- a/src/tidesdb.cpp +++ b/src/tidesdb.cpp @@ -68,6 +68,8 @@ ColumnFamilyConfig ColumnFamilyConfig::defaultConfig() config.minDiskSpace = cConfig.min_disk_space; config.l1FileCountTrigger = cConfig.l1_file_count_trigger; config.l0QueueStallThreshold = cConfig.l0_queue_stall_threshold; + config.tombstoneDensityTrigger = cConfig.tombstone_density_trigger; + config.tombstoneDensityMinEntries = cConfig.tombstone_density_min_entries; config.useBtree = cConfig.use_btree != 0; config.objectLazyCompaction = cConfig.object_lazy_compaction; config.objectPrefetchCompaction = cConfig.object_prefetch_compaction; @@ -104,6 +106,8 @@ ColumnFamilyConfig ColumnFamilyConfig::loadFromIni(const std::string& iniFile, config.minDiskSpace = cConfig.min_disk_space; config.l1FileCountTrigger = cConfig.l1_file_count_trigger; config.l0QueueStallThreshold = cConfig.l0_queue_stall_threshold; + config.tombstoneDensityTrigger = cConfig.tombstone_density_trigger; + config.tombstoneDensityMinEntries = cConfig.tombstone_density_min_entries; config.useBtree = cConfig.use_btree != 0; config.objectLazyCompaction = cConfig.object_lazy_compaction; config.objectPrefetchCompaction = cConfig.object_prefetch_compaction; @@ -136,6 +140,8 @@ void ColumnFamilyConfig::saveToIni(const std::string& iniFile, const std::string cConfig.min_disk_space = config.minDiskSpace; cConfig.l1_file_count_trigger = config.l1FileCountTrigger; cConfig.l0_queue_stall_threshold = config.l0QueueStallThreshold; + cConfig.tombstone_density_trigger = config.tombstoneDensityTrigger; + cConfig.tombstone_density_min_entries = config.tombstoneDensityMinEntries; cConfig.use_btree = config.useBtree ? 1 : 0; cConfig.object_target_file_size = 0; /* retired, reserved in C for ABI compatibility */ cConfig.object_lazy_compaction = config.objectLazyCompaction; @@ -227,6 +233,21 @@ Stats ColumnFamily::getStats() const } } + // Tombstone observability stats + stats.totalTombstones = cStats->total_tombstones; + stats.tombstoneRatio = cStats->tombstone_ratio; + stats.maxSstDensity = cStats->max_sst_density; + stats.maxSstDensityLevel = cStats->max_sst_density_level; + + if (cStats->num_levels > 0 && cStats->level_tombstone_counts != nullptr) + { + stats.levelTombstoneCounts.resize(cStats->num_levels); + for (int i = 0; i < cStats->num_levels; ++i) + { + stats.levelTombstoneCounts[i] = cStats->level_tombstone_counts[i]; + } + } + if (cStats->config != nullptr) { ColumnFamilyConfig cfConfig; @@ -252,6 +273,8 @@ Stats ColumnFamily::getStats() const cfConfig.minDiskSpace = cStats->config->min_disk_space; cfConfig.l1FileCountTrigger = cStats->config->l1_file_count_trigger; cfConfig.l0QueueStallThreshold = cStats->config->l0_queue_stall_threshold; + cfConfig.tombstoneDensityTrigger = cStats->config->tombstone_density_trigger; + cfConfig.tombstoneDensityMinEntries = cStats->config->tombstone_density_min_entries; cfConfig.useBtree = cStats->config->use_btree != 0; cfConfig.objectLazyCompaction = cStats->config->object_lazy_compaction; cfConfig.objectPrefetchCompaction = cStats->config->object_prefetch_compaction; @@ -268,6 +291,52 @@ void ColumnFamily::compact() checkResult(result, "failed to compact column family"); } +void ColumnFamily::compactRange(const std::optional>& startKey, + const std::optional>& endKey) +{ + const uint8_t* startPtr = nullptr; + size_t startSize = 0; + if (startKey.has_value()) + { + startSize = startKey->size(); + startPtr = startSize > 0 ? startKey->data() : nullptr; + } + + const uint8_t* endPtr = nullptr; + size_t endSize = 0; + if (endKey.has_value()) + { + endSize = endKey->size(); + endPtr = endSize > 0 ? endKey->data() : nullptr; + } + + int result = tidesdb_compact_range(cf_, startPtr, startSize, endPtr, endSize); + checkResult(result, "failed to compact range"); +} + +void ColumnFamily::compactRange(const std::optional& startKey, + const std::optional& endKey) +{ + const uint8_t* startPtr = nullptr; + size_t startSize = 0; + if (startKey.has_value()) + { + startSize = startKey->size(); + startPtr = startSize > 0 ? reinterpret_cast(startKey->data()) : nullptr; + } + + const uint8_t* endPtr = nullptr; + size_t endSize = 0; + if (endKey.has_value()) + { + endSize = endKey->size(); + endPtr = endSize > 0 ? reinterpret_cast(endKey->data()) : nullptr; + } + + int result = tidesdb_compact_range(cf_, startPtr, startSize, endPtr, endSize); + checkResult(result, "failed to compact range"); +} + void ColumnFamily::flushMemtable() { int result = tidesdb_flush_memtable(cf_); @@ -351,6 +420,8 @@ void ColumnFamily::updateRuntimeConfig(const ColumnFamilyConfig& config, bool pe cConfig.min_disk_space = config.minDiskSpace; cConfig.l1_file_count_trigger = config.l1FileCountTrigger; cConfig.l0_queue_stall_threshold = config.l0QueueStallThreshold; + cConfig.tombstone_density_trigger = config.tombstoneDensityTrigger; + cConfig.tombstone_density_min_entries = config.tombstoneDensityMinEntries; cConfig.use_btree = config.useBtree ? 1 : 0; cConfig.object_target_file_size = 0; /* retired, reserved in C for ABI compatibility */ cConfig.object_lazy_compaction = config.objectLazyCompaction; @@ -660,6 +731,7 @@ TidesDB::TidesDB(const Config& config) cConfig.unified_memtable_skip_list_probability = config.unifiedMemtableSkipListProbability; cConfig.unified_memtable_sync_mode = static_cast(config.unifiedMemtableSyncMode); cConfig.unified_memtable_sync_interval_us = config.unifiedMemtableSyncIntervalUs; + cConfig.max_concurrent_flushes = config.maxConcurrentFlushes; cConfig.object_store = config.objectStore; tidesdb_objstore_config_t osCfg; @@ -745,6 +817,8 @@ void TidesDB::createColumnFamily(const std::string& name, const ColumnFamilyConf cConfig.min_disk_space = config.minDiskSpace; cConfig.l1_file_count_trigger = config.l1FileCountTrigger; cConfig.l0_queue_stall_threshold = config.l0QueueStallThreshold; + cConfig.tombstone_density_trigger = config.tombstoneDensityTrigger; + cConfig.tombstone_density_min_entries = config.tombstoneDensityMinEntries; cConfig.use_btree = config.useBtree ? 1 : 0; cConfig.object_target_file_size = 0; /* retired, reserved in C for ABI compatibility */ cConfig.object_lazy_compaction = config.objectLazyCompaction; @@ -982,6 +1056,7 @@ Config TidesDB::defaultConfig() config.unifiedMemtableSkipListProbability = cConfig.unified_memtable_skip_list_probability; config.unifiedMemtableSyncMode = static_cast(cConfig.unified_memtable_sync_mode); config.unifiedMemtableSyncIntervalUs = cConfig.unified_memtable_sync_interval_us; + config.maxConcurrentFlushes = cConfig.max_concurrent_flushes; config.objectStore = nullptr; config.objectStoreConfig = std::nullopt; diff --git a/tests/tidesdb_test.cpp b/tests/tidesdb_test.cpp index b2476ad..ca129c5 100644 --- a/tests/tidesdb_test.cpp +++ b/tests/tidesdb_test.cpp @@ -1682,6 +1682,196 @@ TEST_F(TidesDBTest, ColumnFamilyConfigObjectStoreFields) ASSERT_GE(cfConfig.objectPrefetchCompaction, 0); } +TEST_F(TidesDBTest, TombstoneCfConfigRoundTrip) +{ + tidesdb::TidesDB db(getConfig()); + + auto defaults = tidesdb::ColumnFamilyConfig::defaultConfig(); + ASSERT_GT(defaults.tombstoneDensityMinEntries, 0u) + << "default tombstone_density_min_entries should be sourced from C library"; + + auto cfConfig = defaults; + cfConfig.tombstoneDensityTrigger = 0.5; + cfConfig.tombstoneDensityMinEntries = 256; + + db.createColumnFamily("ts_cf", cfConfig); + + auto cf = db.getColumnFamily("ts_cf"); + auto stats = cf.getStats(); + + ASSERT_TRUE(stats.config.has_value()); + ASSERT_DOUBLE_EQ(stats.config->tombstoneDensityTrigger, 0.5); + ASSERT_EQ(stats.config->tombstoneDensityMinEntries, 256u); +} + +TEST_F(TidesDBTest, TombstoneStatsAfterDeletes) +{ + tidesdb::TidesDB db(getConfig()); + + auto cfConfig = tidesdb::ColumnFamilyConfig::defaultConfig(); + cfConfig.writeBufferSize = 1024; // small buffer to make flushes cheap + db.createColumnFamily("ts_cf", cfConfig); + + auto cf = db.getColumnFamily("ts_cf"); + + constexpr int kNumKeys = 200; + + { + auto txn = db.beginTransaction(); + for (int i = 0; i < kNumKeys; ++i) + { + std::string key = "key" + std::to_string(i); + std::string value = "value" + std::to_string(i); + txn.put(cf, key, value, -1); + } + txn.commit(); + } + cf.flushMemtable(); + + { + auto txn = db.beginTransaction(); + for (int i = 0; i < kNumKeys / 2; ++i) + { + std::string key = "key" + std::to_string(i); + txn.del(cf, key); + } + txn.commit(); + } + cf.flushMemtable(); + + // Brief wait for the flush to land + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + + auto stats = cf.getStats(); + + ASSERT_GT(stats.totalTombstones, 0u); + ASSERT_GE(stats.tombstoneRatio, 0.0); + ASSERT_LE(stats.tombstoneRatio, 1.0); + ASSERT_GE(stats.maxSstDensity, 0.0); + ASSERT_LE(stats.maxSstDensity, 1.0); + ASSERT_EQ(stats.levelTombstoneCounts.size(), static_cast(stats.numLevels)); + if (stats.maxSstDensityLevel != 0) + { + ASSERT_GE(stats.maxSstDensityLevel, 1); + ASSERT_LE(stats.maxSstDensityLevel, stats.numLevels); + } +} + +TEST_F(TidesDBTest, CompactRange) +{ + tidesdb::TidesDB db(getConfig()); + + auto cfConfig = tidesdb::ColumnFamilyConfig::defaultConfig(); + cfConfig.writeBufferSize = 1024; // force multiple SSTables + db.createColumnFamily("cr_cf", cfConfig); + + auto cf = db.getColumnFamily("cr_cf"); + + // Several batches of writes + flushes to spread across SSTables/levels + for (int batch = 0; batch < 4; ++batch) + { + auto txn = db.beginTransaction(); + for (int i = 0; i < 100; ++i) + { + std::string key = "k" + std::to_string(batch) + "_" + std::to_string(i); + std::string value = "v" + std::to_string(batch) + "_" + std::to_string(i); + txn.put(cf, key, value, -1); + } + txn.commit(); + cf.flushMemtable(); + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + + // Narrow range compaction should succeed + std::vector startKey{'k', '1', '_', '0'}; + std::vector endKey{'k', '2', '_', '0'}; + ASSERT_NO_THROW(cf.compactRange(startKey, endKey)); + + // Both endpoints unbounded must be rejected + try + { + cf.compactRange(std::optional>{}, + std::optional>{}); + FAIL() << "expected InvalidArgs exception for both-null range"; + } + catch (const tidesdb::Exception& e) + { + ASSERT_EQ(e.code(), tidesdb::ErrorCode::InvalidArgs); + } + + // Keys outside the compacted range must still read back unchanged + { + auto txn = db.beginTransaction(); + auto value = txn.get(cf, "k0_0"); + std::string valueStr(value.begin(), value.end()); + ASSERT_EQ(valueStr, "v0_0"); + + auto value3 = txn.get(cf, "k3_50"); + std::string value3Str(value3.begin(), value3.end()); + ASSERT_EQ(value3Str, "v3_50"); + } +} + +TEST_F(TidesDBTest, CompactRangeStringViewOverload) +{ + tidesdb::TidesDB db(getConfig()); + + auto cfConfig = tidesdb::ColumnFamilyConfig::defaultConfig(); + cfConfig.writeBufferSize = 1024; + db.createColumnFamily("cr_sv_cf", cfConfig); + + auto cf = db.getColumnFamily("cr_sv_cf"); + + { + auto txn = db.beginTransaction(); + for (int i = 0; i < 50; ++i) + { + std::string key = "k_" + std::to_string(i); + std::string value = "v_" + std::to_string(i); + txn.put(cf, key, value, -1); + } + txn.commit(); + } + cf.flushMemtable(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + using OptSV = std::optional; + ASSERT_NO_THROW( + cf.compactRange(OptSV{std::string_view{"k_1"}}, OptSV{std::string_view{"k_3"}})); + + // Unbounded start (nullopt), bounded end -- should succeed + ASSERT_NO_THROW(cf.compactRange(OptSV{}, OptSV{std::string_view{"k_5"}})); +} + +TEST_F(TidesDBTest, MaxConcurrentFlushesConfig) +{ + tidesdb::Config config = getConfig(); + config.maxConcurrentFlushes = 1; + + tidesdb::TidesDB db(config); + + auto cfConfig = tidesdb::ColumnFamilyConfig::defaultConfig(); + db.createColumnFamily("mcf_cf", cfConfig); + + auto cf = db.getColumnFamily("mcf_cf"); + + { + auto txn = db.beginTransaction(); + txn.put(cf, "key", "value", -1); + txn.commit(); + } + ASSERT_NO_THROW(cf.flushMemtable()); +} + +TEST_F(TidesDBTest, MaxConcurrentFlushesDefault) +{ + auto defaultConfig = tidesdb::TidesDB::defaultConfig(); + + // Proves the value is sourced from tidesdb_default_config() rather than zero-initialized + ASSERT_GT(defaultConfig.maxConcurrentFlushes, 0) + << "default max_concurrent_flushes should be sourced from C library defaults"; +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv);