Skip to content

align with tdb 920 with tombstone density capability, and range compa… #53

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
guycipher merged 1 commit into from
Apr 27, 2026
Merged

align with tdb 920 with tombstone density capability, and range compa… #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
cmake_minimum_required(VERSION 3.16)
project(tidesdb_cpp VERSION 2.4.0 LANGUAGES CXX)
project(tidesdb_cpp VERSION 2.5.0 LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
Expand Down
35 changes: 35 additions & 0 deletions include/tidesdb/tidesdb.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,10 @@ struct ColumnFamilyConfig
std::uint64_t minDiskSpace = 100 * 1024 * 1024;
int l1FileCountTrigger = 4;
int l0QueueStallThreshold = 20;
double tombstoneDensityTrigger = 0.0; ///< Per-SSTable tombstone density above which
///< compaction priority escalates (0.0 = disabled)
std::uint64_t tombstoneDensityMinEntries =
1024; ///< SSTables with fewer entries are ignored by the density trigger
bool useBtree = false; ///< Use B+tree format for klog (default: false = block-based)
tidesdb_commit_hook_fn commitHookFn =
nullptr; ///< Optional commit hook callback (runtime-only)
Expand Down Expand Up @@ -275,6 +279,8 @@ struct Config
float unifiedMemtableSkipListProbability = 0; ///< Skip list probability (0 = default 0.25)
SyncMode unifiedMemtableSyncMode = SyncMode::None; ///< Sync mode for unified WAL
std::uint64_t unifiedMemtableSyncIntervalUs = 0; ///< Sync interval for unified WAL
int maxConcurrentFlushes =
0; ///< Global cap on in-flight memtable flushes across all CFs (0 = library default)
tidesdb_objstore_t* objectStore =
nullptr; ///< Pluggable object store connector (nullptr = local only)
std::optional<ObjectStoreConfig>
Expand Down Expand Up @@ -302,6 +308,12 @@ struct Stats
std::uint64_t btreeTotalNodes = 0; ///< Total B+tree nodes across all SSTables
std::uint32_t btreeMaxHeight = 0; ///< Maximum tree height across all SSTables
double btreeAvgHeight = 0.0; ///< Average tree height across all SSTables
std::uint64_t totalTombstones = 0; ///< Sum of tombstone counts across all SSTables
double tombstoneRatio = 0.0; ///< total_tombstones / total_keys (0.0 if total_keys == 0)
std::vector<std::uint64_t>
levelTombstoneCounts; ///< Per-level tombstone counts (parallels levelKeyCounts)
double maxSstDensity = 0.0; ///< Worst per-SSTable tombstone density observed (0.0 to 1.0)
int maxSstDensityLevel = 0; ///< 1-based level where maxSstDensity was observed (0 if none)
};

/**
Expand Down Expand Up @@ -378,6 +390,29 @@ class ColumnFamily
*/
void compact();

/**
* @brief Synchronously compact only SSTables overlapping [startKey, endKey)
*
* Blocks the calling thread until the merge commits or fails. Pass an empty
* optional (std::nullopt) for either endpoint to leave that side unbounded.
* Both endpoints unbounded is rejected with ErrorCode::InvalidArgs - use
* compact() for a full column-family compaction.
*
* @param startKey Start of range (inclusive); std::nullopt = unbounded
* @param endKey End of range (exclusive); std::nullopt = unbounded
*/
void compactRange(const std::optional<std::vector<std::uint8_t>>& startKey,
const std::optional<std::vector<std::uint8_t>>& endKey);

/**
* @brief compactRange overload taking string_view bounds
*
* Pass empty optional for an unbounded endpoint. Empty (but present)
* string_views are forwarded as zero-length keys, matching the C API.
*/
void compactRange(const std::optional<std::string_view>& startKey,
const std::optional<std::string_view>& endKey);

/**
* @brief Manually trigger memtable flush
*/
Expand Down
75 changes: 75 additions & 0 deletions src/tidesdb.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ ColumnFamilyConfig ColumnFamilyConfig::defaultConfig()
config.minDiskSpace = cConfig.min_disk_space;
config.l1FileCountTrigger = cConfig.l1_file_count_trigger;
config.l0QueueStallThreshold = cConfig.l0_queue_stall_threshold;
config.tombstoneDensityTrigger = cConfig.tombstone_density_trigger;
config.tombstoneDensityMinEntries = cConfig.tombstone_density_min_entries;
config.useBtree = cConfig.use_btree != 0;
config.objectLazyCompaction = cConfig.object_lazy_compaction;
config.objectPrefetchCompaction = cConfig.object_prefetch_compaction;
Expand Down Expand Up @@ -104,6 +106,8 @@ ColumnFamilyConfig ColumnFamilyConfig::loadFromIni(const std::string& iniFile,
config.minDiskSpace = cConfig.min_disk_space;
config.l1FileCountTrigger = cConfig.l1_file_count_trigger;
config.l0QueueStallThreshold = cConfig.l0_queue_stall_threshold;
config.tombstoneDensityTrigger = cConfig.tombstone_density_trigger;
config.tombstoneDensityMinEntries = cConfig.tombstone_density_min_entries;
config.useBtree = cConfig.use_btree != 0;
config.objectLazyCompaction = cConfig.object_lazy_compaction;
config.objectPrefetchCompaction = cConfig.object_prefetch_compaction;
Expand Down Expand Up @@ -136,6 +140,8 @@ void ColumnFamilyConfig::saveToIni(const std::string& iniFile, const std::string
cConfig.min_disk_space = config.minDiskSpace;
cConfig.l1_file_count_trigger = config.l1FileCountTrigger;
cConfig.l0_queue_stall_threshold = config.l0QueueStallThreshold;
cConfig.tombstone_density_trigger = config.tombstoneDensityTrigger;
cConfig.tombstone_density_min_entries = config.tombstoneDensityMinEntries;
cConfig.use_btree = config.useBtree ? 1 : 0;
cConfig.object_target_file_size = 0; /* retired, reserved in C for ABI compatibility */
cConfig.object_lazy_compaction = config.objectLazyCompaction;
Expand Down Expand Up @@ -227,6 +233,21 @@ Stats ColumnFamily::getStats() const
}
}

// Tombstone observability stats
stats.totalTombstones = cStats->total_tombstones;
stats.tombstoneRatio = cStats->tombstone_ratio;
stats.maxSstDensity = cStats->max_sst_density;
stats.maxSstDensityLevel = cStats->max_sst_density_level;

if (cStats->num_levels > 0 && cStats->level_tombstone_counts != nullptr)
{
stats.levelTombstoneCounts.resize(cStats->num_levels);
for (int i = 0; i < cStats->num_levels; ++i)
{
stats.levelTombstoneCounts[i] = cStats->level_tombstone_counts[i];
}
}

if (cStats->config != nullptr)
{
ColumnFamilyConfig cfConfig;
Expand All @@ -252,6 +273,8 @@ Stats ColumnFamily::getStats() const
cfConfig.minDiskSpace = cStats->config->min_disk_space;
cfConfig.l1FileCountTrigger = cStats->config->l1_file_count_trigger;
cfConfig.l0QueueStallThreshold = cStats->config->l0_queue_stall_threshold;
cfConfig.tombstoneDensityTrigger = cStats->config->tombstone_density_trigger;
cfConfig.tombstoneDensityMinEntries = cStats->config->tombstone_density_min_entries;
cfConfig.useBtree = cStats->config->use_btree != 0;
cfConfig.objectLazyCompaction = cStats->config->object_lazy_compaction;
cfConfig.objectPrefetchCompaction = cStats->config->object_prefetch_compaction;
Expand All @@ -268,6 +291,52 @@ void ColumnFamily::compact()
checkResult(result, "failed to compact column family");
}

void ColumnFamily::compactRange(const std::optional<std::vector<std::uint8_t>>& startKey,
const std::optional<std::vector<std::uint8_t>>& endKey)
{
const uint8_t* startPtr = nullptr;
size_t startSize = 0;
if (startKey.has_value())
{
startSize = startKey->size();
startPtr = startSize > 0 ? startKey->data() : nullptr;
}

const uint8_t* endPtr = nullptr;
size_t endSize = 0;
if (endKey.has_value())
{
endSize = endKey->size();
endPtr = endSize > 0 ? endKey->data() : nullptr;
}

int result = tidesdb_compact_range(cf_, startPtr, startSize, endPtr, endSize);
checkResult(result, "failed to compact range");
}

void ColumnFamily::compactRange(const std::optional<std::string_view>& startKey,
const std::optional<std::string_view>& endKey)
{
const uint8_t* startPtr = nullptr;
size_t startSize = 0;
if (startKey.has_value())
{
startSize = startKey->size();
startPtr = startSize > 0 ? reinterpret_cast<const uint8_t*>(startKey->data()) : nullptr;
}

const uint8_t* endPtr = nullptr;
size_t endSize = 0;
if (endKey.has_value())
{
endSize = endKey->size();
endPtr = endSize > 0 ? reinterpret_cast<const uint8_t*>(endKey->data()) : nullptr;
}

int result = tidesdb_compact_range(cf_, startPtr, startSize, endPtr, endSize);
checkResult(result, "failed to compact range");
}

void ColumnFamily::flushMemtable()
{
int result = tidesdb_flush_memtable(cf_);
Expand Down Expand Up @@ -351,6 +420,8 @@ void ColumnFamily::updateRuntimeConfig(const ColumnFamilyConfig& config, bool pe
cConfig.min_disk_space = config.minDiskSpace;
cConfig.l1_file_count_trigger = config.l1FileCountTrigger;
cConfig.l0_queue_stall_threshold = config.l0QueueStallThreshold;
cConfig.tombstone_density_trigger = config.tombstoneDensityTrigger;
cConfig.tombstone_density_min_entries = config.tombstoneDensityMinEntries;
cConfig.use_btree = config.useBtree ? 1 : 0;
cConfig.object_target_file_size = 0; /* retired, reserved in C for ABI compatibility */
cConfig.object_lazy_compaction = config.objectLazyCompaction;
Expand Down Expand Up @@ -660,6 +731,7 @@ TidesDB::TidesDB(const Config& config)
cConfig.unified_memtable_skip_list_probability = config.unifiedMemtableSkipListProbability;
cConfig.unified_memtable_sync_mode = static_cast<int>(config.unifiedMemtableSyncMode);
cConfig.unified_memtable_sync_interval_us = config.unifiedMemtableSyncIntervalUs;
cConfig.max_concurrent_flushes = config.maxConcurrentFlushes;
cConfig.object_store = config.objectStore;

tidesdb_objstore_config_t osCfg;
Expand Down Expand Up @@ -745,6 +817,8 @@ void TidesDB::createColumnFamily(const std::string& name, const ColumnFamilyConf
cConfig.min_disk_space = config.minDiskSpace;
cConfig.l1_file_count_trigger = config.l1FileCountTrigger;
cConfig.l0_queue_stall_threshold = config.l0QueueStallThreshold;
cConfig.tombstone_density_trigger = config.tombstoneDensityTrigger;
cConfig.tombstone_density_min_entries = config.tombstoneDensityMinEntries;
cConfig.use_btree = config.useBtree ? 1 : 0;
cConfig.object_target_file_size = 0; /* retired, reserved in C for ABI compatibility */
cConfig.object_lazy_compaction = config.objectLazyCompaction;
Expand Down Expand Up @@ -982,6 +1056,7 @@ Config TidesDB::defaultConfig()
config.unifiedMemtableSkipListProbability = cConfig.unified_memtable_skip_list_probability;
config.unifiedMemtableSyncMode = static_cast<SyncMode>(cConfig.unified_memtable_sync_mode);
config.unifiedMemtableSyncIntervalUs = cConfig.unified_memtable_sync_interval_us;
config.maxConcurrentFlushes = cConfig.max_concurrent_flushes;
config.objectStore = nullptr;
config.objectStoreConfig = std::nullopt;

Expand Down
Loading
Loading