From 7bac7eedfd6bbe24bf27e88e0fbb213a74972939 Mon Sep 17 00:00:00 2001 From: Alex Gaetano Padula Date: Sun, 25 Jan 2026 20:13:12 -0500 Subject: [PATCH] align lua library with tidesdb v7.4.0 and all features --- src/tidesdb.lua | 149 +++++++++++++++++- tests/test_tidesdb.lua | 124 +++++++++++++++ ...2.0-1.rockspec => tidesdb-0.3.0-1.rockspec | 4 +- 3 files changed, 271 insertions(+), 6 deletions(-) rename tidesdb-0.2.0-1.rockspec => tidesdb-0.3.0-1.rockspec (95%) diff --git a/src/tidesdb.lua b/src/tidesdb.lua index 9649d33..93977b3 100644 --- a/src/tidesdb.lua +++ b/src/tidesdb.lua @@ -46,7 +46,7 @@ ffi.cdef[[ int min_levels; int dividing_level_offset; size_t klog_value_threshold; - int compression_algo; + int compression_algorithm; int enable_bloom_filter; double bloom_fpr; int enable_block_indexes; @@ -81,6 +81,13 @@ ffi.cdef[[ size_t* level_sizes; int* level_num_sstables; tidesdb_column_family_config_t* config; + uint64_t total_keys; + uint64_t total_data_size; + double avg_key_size; + double avg_value_size; + uint64_t* level_key_counts; + double read_amp; + double hit_rate; } tidesdb_stats_t; typedef struct { @@ -102,6 +109,7 @@ ffi.cdef[[ // Column family functions int tidesdb_create_column_family(void* db, const char* name, tidesdb_column_family_config_t* config); int tidesdb_drop_column_family(void* db, const char* name); + int tidesdb_rename_column_family(void* db, const char* old_name, const char* new_name); void* tidesdb_get_column_family(void* db, const char* name); int tidesdb_list_column_families(void* db, char*** names, int* count); @@ -134,10 +142,36 @@ ffi.cdef[[ // Column family operations int tidesdb_compact(void* cf); int tidesdb_flush_memtable(void* cf); + int tidesdb_is_flushing(void* cf); + int tidesdb_is_compacting(void* cf); int tidesdb_get_stats(void* cf, tidesdb_stats_t** stats); void tidesdb_free_stats(tidesdb_stats_t* stats); int tidesdb_get_cache_stats(void* db, tidesdb_cache_stats_t* stats); + // Backup operations + int tidesdb_backup(void* db, const char* dir); + + // Configuration operations + int tidesdb_cf_config_load_from_ini(const char* ini_file, const char* section_name, tidesdb_column_family_config_t* config); + int tidesdb_cf_config_save_to_ini(const char* ini_file, const char* section_name, tidesdb_column_family_config_t* config); + int tidesdb_cf_update_runtime_config(void* cf, tidesdb_column_family_config_t* new_config, int persist_to_disk); + + // Comparator operations + typedef int (*tidesdb_comparator_fn)(const uint8_t* key1, size_t key1_size, const uint8_t* key2, size_t key2_size, void* ctx); + int tidesdb_register_comparator(void* db, const char* name, tidesdb_comparator_fn fn, const char* ctx_str, void* ctx); + int tidesdb_get_comparator(void* db, const char* name, tidesdb_comparator_fn* fn, void** ctx); + + // Built-in comparator functions + int tidesdb_comparator_memcmp(const uint8_t* key1, size_t key1_size, const uint8_t* key2, size_t key2_size, void* ctx); + int tidesdb_comparator_lexicographic(const uint8_t* key1, size_t key1_size, const uint8_t* key2, size_t key2_size, void* ctx); + int tidesdb_comparator_uint64(const uint8_t* key1, size_t key1_size, const uint8_t* key2, size_t key2_size, void* ctx); + int tidesdb_comparator_int64(const uint8_t* key1, size_t key1_size, const uint8_t* key2, size_t key2_size, void* ctx); + int tidesdb_comparator_reverse_memcmp(const uint8_t* key1, size_t key1_size, const uint8_t* key2, size_t key2_size, void* ctx); + int tidesdb_comparator_case_insensitive(const uint8_t* key1, size_t key1_size, const uint8_t* key2, size_t key2_size, void* ctx); + + // Memory management + void tidesdb_free(void* ptr); + // C standard library void free(void* ptr); ]] @@ -305,7 +339,7 @@ function tidesdb.default_column_family_config() min_levels = c_config.min_levels, dividing_level_offset = c_config.dividing_level_offset, klog_value_threshold = tonumber(c_config.klog_value_threshold), - compression_algorithm = c_config.compression_algo, + compression_algorithm = c_config.compression_algorithm, enable_bloom_filter = c_config.enable_bloom_filter ~= 0, bloom_fpr = c_config.bloom_fpr, enable_block_indexes = c_config.enable_block_indexes ~= 0, @@ -331,7 +365,7 @@ local function config_to_c_struct(config) c_config.min_levels = config.min_levels or 5 c_config.dividing_level_offset = config.dividing_level_offset or 2 c_config.klog_value_threshold = config.klog_value_threshold or 512 - c_config.compression_algo = config.compression_algorithm or tidesdb.CompressionAlgorithm.LZ4_COMPRESSION + c_config.compression_algorithm = config.compression_algorithm or tidesdb.CompressionAlgorithm.LZ4_COMPRESSION c_config.enable_bloom_filter = config.enable_bloom_filter and 1 or 0 c_config.bloom_fpr = config.bloom_fpr or 0.01 c_config.enable_block_indexes = config.enable_block_indexes and 1 or 0 @@ -476,6 +510,23 @@ function ColumnFamily:flush_memtable() check_result(result, "failed to flush memtable") end +function ColumnFamily:is_flushing() + return lib.tidesdb_is_flushing(self._cf) ~= 0 +end + +function ColumnFamily:is_compacting() + return lib.tidesdb_is_compacting(self._cf) ~= 0 +end + +function ColumnFamily:update_runtime_config(config, persist_to_disk) + if persist_to_disk == nil then + persist_to_disk = true + end + local c_config = config_to_c_struct(config) + local result = lib.tidesdb_cf_update_runtime_config(self._cf, c_config, persist_to_disk and 1 or 0) + check_result(result, "failed to update runtime config") +end + function ColumnFamily:get_stats() local stats_ptr = ffi.new("tidesdb_stats_t*[1]") local result = lib.tidesdb_get_stats(self._cf, stats_ptr) @@ -507,7 +558,7 @@ function ColumnFamily:get_stats() min_levels = c_cfg.min_levels, dividing_level_offset = c_cfg.dividing_level_offset, klog_value_threshold = tonumber(c_cfg.klog_value_threshold), - compression_algorithm = c_cfg.compression_algo, + compression_algorithm = c_cfg.compression_algorithm, enable_bloom_filter = c_cfg.enable_bloom_filter ~= 0, bloom_fpr = c_cfg.bloom_fpr, enable_block_indexes = c_cfg.enable_block_indexes ~= 0, @@ -525,12 +576,26 @@ function ColumnFamily:get_stats() } end + local level_key_counts = {} + if c_stats.num_levels > 0 and c_stats.level_key_counts ~= nil then + for i = 0, c_stats.num_levels - 1 do + table.insert(level_key_counts, tonumber(c_stats.level_key_counts[i])) + end + end + local stats = { num_levels = c_stats.num_levels, memtable_size = tonumber(c_stats.memtable_size), level_sizes = level_sizes, level_num_sstables = level_num_sstables, config = config, + total_keys = tonumber(c_stats.total_keys), + total_data_size = tonumber(c_stats.total_data_size), + avg_key_size = c_stats.avg_key_size, + avg_value_size = c_stats.avg_value_size, + level_key_counts = level_key_counts, + read_amp = c_stats.read_amp, + hit_rate = c_stats.hit_rate, } lib.tidesdb_free_stats(stats_ptr[0]) @@ -761,6 +826,15 @@ function TidesDB:drop_column_family(name) check_result(result, "failed to drop column family") end +function TidesDB:rename_column_family(old_name, new_name) + if self._closed then + error(TidesDBError.new("Database is closed")) + end + + local result = lib.tidesdb_rename_column_family(self._db, old_name, new_name) + check_result(result, "failed to rename column family") +end + function TidesDB:get_column_family(name) if self._closed then error(TidesDBError.new("Database is closed")) @@ -846,8 +920,75 @@ function TidesDB:get_cache_stats() } end +function TidesDB:backup(dir) + if self._closed then + error(TidesDBError.new("Database is closed")) + end + + local result = lib.tidesdb_backup(self._db, dir) + check_result(result, "failed to create backup") +end + +function TidesDB:register_comparator(name, fn, ctx_str, ctx) + if self._closed then + error(TidesDBError.new("Database is closed")) + end + + local result = lib.tidesdb_register_comparator(self._db, name, fn, ctx_str, ctx) + check_result(result, "failed to register comparator") +end + +function TidesDB:get_comparator(name) + if self._closed then + error(TidesDBError.new("Database is closed")) + end + + local fn_ptr = ffi.new("tidesdb_comparator_fn[1]") + local ctx_ptr = ffi.new("void*[1]") + local result = lib.tidesdb_get_comparator(self._db, name, fn_ptr, ctx_ptr) + check_result(result, "failed to get comparator") + + return fn_ptr[0], ctx_ptr[0] +end + tidesdb.TidesDB = TidesDB +-- Configuration file operations +function tidesdb.load_config_from_ini(ini_file, section_name) + local c_config = ffi.new("tidesdb_column_family_config_t") + local result = lib.tidesdb_cf_config_load_from_ini(ini_file, section_name, c_config) + check_result(result, "failed to load config from INI") + + return { + write_buffer_size = tonumber(c_config.write_buffer_size), + level_size_ratio = tonumber(c_config.level_size_ratio), + min_levels = c_config.min_levels, + dividing_level_offset = c_config.dividing_level_offset, + klog_value_threshold = tonumber(c_config.klog_value_threshold), + compression_algorithm = c_config.compression_algorithm, + enable_bloom_filter = c_config.enable_bloom_filter ~= 0, + bloom_fpr = c_config.bloom_fpr, + enable_block_indexes = c_config.enable_block_indexes ~= 0, + index_sample_ratio = c_config.index_sample_ratio, + block_index_prefix_len = c_config.block_index_prefix_len, + sync_mode = c_config.sync_mode, + sync_interval_us = tonumber(c_config.sync_interval_us), + comparator_name = ffi.string(c_config.comparator_name), + skip_list_max_level = c_config.skip_list_max_level, + skip_list_probability = c_config.skip_list_probability, + default_isolation_level = c_config.default_isolation_level, + min_disk_space = tonumber(c_config.min_disk_space), + l1_file_count_trigger = c_config.l1_file_count_trigger, + l0_queue_stall_threshold = c_config.l0_queue_stall_threshold, + } +end + +function tidesdb.save_config_to_ini(ini_file, section_name, config) + local c_config = config_to_c_struct(config) + local result = lib.tidesdb_cf_config_save_to_ini(ini_file, section_name, c_config) + check_result(result, "failed to save config to INI") +end + -- Version tidesdb._VERSION = "0.2.0" diff --git a/tests/test_tidesdb.lua b/tests/test_tidesdb.lua index 3211771..bf0b97d 100644 --- a/tests/test_tidesdb.lua +++ b/tests/test_tidesdb.lua @@ -268,6 +268,13 @@ function tests.test_stats() local stats = cf:get_stats() assert_true(stats.num_levels >= 0, "num_levels should be >= 0") assert_true(stats.memtable_size >= 0, "memtable_size should be >= 0") + -- Test new stats fields + assert_true(stats.total_keys ~= nil, "total_keys should exist") + assert_true(stats.total_data_size ~= nil, "total_data_size should exist") + assert_true(stats.avg_key_size ~= nil, "avg_key_size should exist") + assert_true(stats.avg_value_size ~= nil, "avg_value_size should exist") + assert_true(stats.read_amp ~= nil, "read_amp should exist") + assert_true(stats.hit_rate ~= nil, "hit_rate should exist") -- Get cache stats local cache_stats = db:get_cache_stats() @@ -279,6 +286,123 @@ function tests.test_stats() print("PASS: test_stats") end +function tests.test_rename_column_family() + local path = "./test_db_rename_cf" + cleanup_db(path) + + local db = tidesdb.TidesDB.open(path) + db:create_column_family("old_cf") + + -- Insert data + local cf = db:get_column_family("old_cf") + local txn = db:begin_txn() + txn:put(cf, "key1", "value1") + txn:commit() + txn:free() + + -- Rename column family + db:rename_column_family("old_cf", "new_cf") + + -- Verify old name doesn't exist + local err = assert_error(function() + db:get_column_family("old_cf") + end, "old_cf should not exist after rename") + + -- Verify new name exists and data is preserved + local new_cf = db:get_column_family("new_cf") + local read_txn = db:begin_txn() + local value = read_txn:get(new_cf, "key1") + assert_eq(value, "value1", "data should be preserved after rename") + read_txn:free() + + db:drop_column_family("new_cf") + db:close() + cleanup_db(path) + print("PASS: test_rename_column_family") +end + +function tests.test_is_flushing_compacting() + local path = "./test_db_flush_compact" + cleanup_db(path) + + local db = tidesdb.TidesDB.open(path) + db:create_column_family("test_cf") + local cf = db:get_column_family("test_cf") + + -- Check status (should be false when idle) + local is_flushing = cf:is_flushing() + local is_compacting = cf:is_compacting() + assert_true(is_flushing == false or is_flushing == true, "is_flushing should return boolean") + assert_true(is_compacting == false or is_compacting == true, "is_compacting should return boolean") + + db:drop_column_family("test_cf") + db:close() + cleanup_db(path) + print("PASS: test_is_flushing_compacting") +end + +function tests.test_backup() + local path = "./test_db_backup" + local backup_path = "./test_db_backup_copy" + cleanup_db(path) + cleanup_db(backup_path) + + local db = tidesdb.TidesDB.open(path) + db:create_column_family("test_cf") + local cf = db:get_column_family("test_cf") + + -- Insert data + local txn = db:begin_txn() + txn:put(cf, "key1", "value1") + txn:commit() + txn:free() + + -- Create backup + db:backup(backup_path) + + db:close() + + -- Open backup and verify data + local backup_db = tidesdb.TidesDB.open(backup_path) + local backup_cf = backup_db:get_column_family("test_cf") + local read_txn = backup_db:begin_txn() + local value = read_txn:get(backup_cf, "key1") + assert_eq(value, "value1", "backup should contain original data") + read_txn:free() + + backup_db:close() + cleanup_db(path) + cleanup_db(backup_path) + print("PASS: test_backup") +end + +function tests.test_update_runtime_config() + local path = "./test_db_runtime_config" + cleanup_db(path) + + local db = tidesdb.TidesDB.open(path) + db:create_column_family("test_cf") + local cf = db:get_column_family("test_cf") + + -- Get current config + local stats = cf:get_stats() + local original_write_buffer_size = stats.config.write_buffer_size + + -- Update runtime config + local new_config = tidesdb.default_column_family_config() + new_config.write_buffer_size = 128 * 1024 * 1024 -- 128MB + cf:update_runtime_config(new_config, false) -- don't persist + + -- Verify config was updated + local new_stats = cf:get_stats() + assert_eq(new_stats.config.write_buffer_size, 128 * 1024 * 1024, "write_buffer_size should be updated") + + db:drop_column_family("test_cf") + db:close() + cleanup_db(path) + print("PASS: test_update_runtime_config") +end + -- Run all tests local function run_tests() print("Running TidesDB Lua tests...") diff --git a/tidesdb-0.2.0-1.rockspec b/tidesdb-0.3.0-1.rockspec similarity index 95% rename from tidesdb-0.2.0-1.rockspec rename to tidesdb-0.3.0-1.rockspec index 40eb863..e0737c2 100644 --- a/tidesdb-0.2.0-1.rockspec +++ b/tidesdb-0.3.0-1.rockspec @@ -1,8 +1,8 @@ package = "tidesdb" -version = "0.2.0-1" +version = "0.3.0-1" source = { url = "git://github.com/tidesdb/tidesdb-lua.git", - tag = "v0.2.0" + tag = "v0.3.0" } description = { summary = "Official Lua bindings for TidesDB - A high-performance embedded key-value storage engine",