diff --git a/src/tidesdb.lua b/src/tidesdb.lua index 2d49ef3..8a8afbd 100644 --- a/src/tidesdb.lua +++ b/src/tidesdb.lua @@ -88,6 +88,26 @@ ffi.cdef[[ int object_prefetch_compaction; } tidesdb_column_family_config_t; + // Object store configuration + typedef struct { + const char *local_cache_path; + size_t local_cache_max_bytes; + int cache_on_read; + int cache_on_write; + int max_concurrent_uploads; + int max_concurrent_downloads; + size_t multipart_threshold; + size_t multipart_part_size; + int sync_manifest_to_object; + int replicate_wal; + int wal_upload_sync; + size_t wal_sync_threshold_bytes; + int wal_sync_on_commit; + int replica_mode; + uint64_t replica_sync_interval_us; + int replica_replay_wal; + } tidesdb_objstore_config_t; + typedef struct { const char* db_path; int num_flush_threads; @@ -105,7 +125,7 @@ ffi.cdef[[ int unified_memtable_sync_mode; uint64_t unified_memtable_sync_interval_us; void* object_store; - void* object_store_config; + tidesdb_objstore_config_t* object_store_config; } tidesdb_config_t; typedef struct { @@ -137,6 +157,10 @@ ffi.cdef[[ size_t num_partitions; } tidesdb_cache_stats_t; + // Object store functions + tidesdb_objstore_config_t tidesdb_objstore_default_config(void); + void* tidesdb_objstore_fs_create(const char* root_dir); + // Database functions tidesdb_column_family_config_t tidesdb_default_column_family_config(void); tidesdb_config_t tidesdb_default_config(void); @@ -466,6 +490,59 @@ function tidesdb.default_column_family_config() } end +-- Object store configuration +function tidesdb.default_objstore_config() + local c_config = lib.tidesdb_objstore_default_config() + return { + local_cache_path = c_config.local_cache_path ~= nil and ffi.string(c_config.local_cache_path) or nil, + local_cache_max_bytes = tonumber(c_config.local_cache_max_bytes), + cache_on_read = c_config.cache_on_read ~= 0, + cache_on_write = c_config.cache_on_write ~= 0, + max_concurrent_uploads = c_config.max_concurrent_uploads, + max_concurrent_downloads = c_config.max_concurrent_downloads, + multipart_threshold = tonumber(c_config.multipart_threshold), + multipart_part_size = tonumber(c_config.multipart_part_size), + sync_manifest_to_object = c_config.sync_manifest_to_object ~= 0, + replicate_wal = c_config.replicate_wal ~= 0, + wal_upload_sync = c_config.wal_upload_sync ~= 0, + wal_sync_threshold_bytes = tonumber(c_config.wal_sync_threshold_bytes), + wal_sync_on_commit = c_config.wal_sync_on_commit ~= 0, + replica_mode = c_config.replica_mode ~= 0, + replica_sync_interval_us = tonumber(c_config.replica_sync_interval_us), + replica_replay_wal = c_config.replica_replay_wal ~= 0, + } +end + +-- Convert Lua objstore config to C struct +local function objstore_config_to_c_struct(config) + local c_config = ffi.new("tidesdb_objstore_config_t") + c_config.local_cache_path = config.local_cache_path + c_config.local_cache_max_bytes = config.local_cache_max_bytes or 0 + c_config.cache_on_read = (config.cache_on_read == nil or config.cache_on_read) and 1 or 0 + c_config.cache_on_write = (config.cache_on_write == nil or config.cache_on_write) and 1 or 0 + c_config.max_concurrent_uploads = config.max_concurrent_uploads or 4 + c_config.max_concurrent_downloads = config.max_concurrent_downloads or 8 + c_config.multipart_threshold = config.multipart_threshold or 67108864 + c_config.multipart_part_size = config.multipart_part_size or 8388608 + c_config.sync_manifest_to_object = (config.sync_manifest_to_object == nil or config.sync_manifest_to_object) and 1 or 0 + c_config.replicate_wal = (config.replicate_wal == nil or config.replicate_wal) and 1 or 0 + c_config.wal_upload_sync = config.wal_upload_sync and 1 or 0 + c_config.wal_sync_threshold_bytes = config.wal_sync_threshold_bytes or 1048576 + c_config.wal_sync_on_commit = config.wal_sync_on_commit and 1 or 0 + c_config.replica_mode = config.replica_mode and 1 or 0 + c_config.replica_sync_interval_us = config.replica_sync_interval_us or 5000000 + c_config.replica_replay_wal = (config.replica_replay_wal == nil or config.replica_replay_wal) and 1 or 0 + return c_config +end + +function tidesdb.objstore_fs_create(root_dir) + local store = lib.tidesdb_objstore_fs_create(root_dir) + if store == nil then + error(TidesDBError.new("failed to create filesystem object store connector")) + end + return store +end + -- Convert Lua config to C struct local function config_to_c_struct(config, cf_name) local c_config = ffi.new("tidesdb_column_family_config_t") @@ -957,6 +1034,17 @@ function TidesDB.new(config) c_config.unified_memtable_sync_mode = config.unified_memtable_sync_mode or tidesdb.SyncMode.SYNC_INTERVAL c_config.unified_memtable_sync_interval_us = config.unified_memtable_sync_interval_us or 128000 + -- Object store configuration + if config.object_store then + c_config.object_store = config.object_store + end + + local os_config_holder + if config.object_store_config then + os_config_holder = objstore_config_to_c_struct(config.object_store_config) + c_config.object_store_config = os_config_holder + end + local db_ptr = ffi.new("void*[1]") local result = lib.tidesdb_open(c_config, db_ptr) check_result(result, "failed to open database") @@ -983,6 +1071,8 @@ function TidesDB.open(path, options) unified_memtable_skip_list_probability = options.unified_memtable_skip_list_probability, unified_memtable_sync_mode = options.unified_memtable_sync_mode, unified_memtable_sync_interval_us = options.unified_memtable_sync_interval_us, + object_store = options.object_store, + object_store_config = options.object_store_config, } return TidesDB.new(config) end @@ -1277,6 +1367,6 @@ function tidesdb.save_config_to_ini(ini_file, section_name, config) end -- Version -tidesdb._VERSION = "0.5.7" +tidesdb._VERSION = "0.5.8" return tidesdb diff --git a/tests/test_tidesdb.lua b/tests/test_tidesdb.lua index 1256824..62ea86e 100644 --- a/tests/test_tidesdb.lua +++ b/tests/test_tidesdb.lua @@ -1232,6 +1232,90 @@ function tests.test_db_stats_extended_fields() print("PASS: test_db_stats_extended_fields") end +function tests.test_objstore_config_defaults() + -- Test default object store config has correct fields and defaults + local os_config = tidesdb.default_objstore_config() + + assert_true(os_config.local_cache_max_bytes ~= nil, "local_cache_max_bytes should exist") + assert_eq(os_config.local_cache_max_bytes, 0, "local_cache_max_bytes should default to 0") + assert_eq(os_config.cache_on_read, true, "cache_on_read should default to true") + assert_eq(os_config.cache_on_write, true, "cache_on_write should default to true") + assert_eq(os_config.max_concurrent_uploads, 4, "max_concurrent_uploads should default to 4") + assert_eq(os_config.max_concurrent_downloads, 8, "max_concurrent_downloads should default to 8") + assert_true(os_config.multipart_threshold > 0, "multipart_threshold should be > 0") + assert_true(os_config.multipart_part_size > 0, "multipart_part_size should be > 0") + assert_eq(os_config.sync_manifest_to_object, true, "sync_manifest_to_object should default to true") + assert_eq(os_config.replicate_wal, true, "replicate_wal should default to true") + assert_eq(os_config.wal_upload_sync, false, "wal_upload_sync should default to false") + assert_true(os_config.wal_sync_threshold_bytes > 0, "wal_sync_threshold_bytes should be > 0") + assert_eq(os_config.wal_sync_on_commit, false, "wal_sync_on_commit should default to false") + assert_eq(os_config.replica_mode, false, "replica_mode should default to false") + assert_true(os_config.replica_sync_interval_us > 0, "replica_sync_interval_us should be > 0") + assert_eq(os_config.replica_replay_wal, true, "replica_replay_wal should default to true") + + print("PASS: test_objstore_config_defaults") +end + +function tests.test_objstore_fs_create() + local path = "./test_objstore_root" + os.execute("rm -rf " .. path) + os.execute("mkdir -p " .. path) + + -- Create filesystem connector + local store = tidesdb.objstore_fs_create(path) + assert_true(store ~= nil, "filesystem object store connector should be created") + + os.execute("rm -rf " .. path) + print("PASS: test_objstore_fs_create") +end + +function tests.test_objstore_open_with_fs_connector() + local db_path = "./test_db_objstore" + local store_path = "./test_objstore_data" + local cleanup_db = function(p) os.execute("rm -rf " .. p) end + cleanup_db(db_path) + cleanup_db(store_path) + os.execute("mkdir -p " .. store_path) + + -- Create filesystem connector and config + local store = tidesdb.objstore_fs_create(store_path) + local os_config = tidesdb.default_objstore_config() + os_config.local_cache_max_bytes = 128 * 1024 * 1024 + os_config.max_concurrent_uploads = 2 + + -- Open database with object store + local db = tidesdb.TidesDB.open(db_path, { + log_level = tidesdb.LogLevel.LOG_WARN, + object_store = store, + object_store_config = os_config, + }) + assert_true(db ~= nil, "database should open with object store connector") + + -- Basic operations should work + db:create_column_family("test_cf") + local cf = db:get_column_family("test_cf") + + local txn = db:begin_txn() + txn:put(cf, "key1", "value1") + txn:commit() + txn:free() + + local read_txn = db:begin_txn() + local v = read_txn:get(cf, "key1") + assert_eq(v, "value1", "should read value with object store enabled") + read_txn:free() + + -- Verify object store stats reflect enabled state + local db_stats = db:get_db_stats() + assert_eq(db_stats.object_store_enabled, true, "object_store_enabled should be true") + + db:drop_column_family("test_cf") + db:close() + cleanup_db(db_path) + cleanup_db(store_path) + print("PASS: test_objstore_open_with_fs_connector") +end + function tests.test_promote_to_primary() local path = "./test_db_promote" cleanup_db(path) diff --git a/tidesdb-0.5.7-1.rockspec b/tidesdb-0.5.8-1.rockspec similarity index 95% rename from tidesdb-0.5.7-1.rockspec rename to tidesdb-0.5.8-1.rockspec index 4599b41..934483a 100644 --- a/tidesdb-0.5.7-1.rockspec +++ b/tidesdb-0.5.8-1.rockspec @@ -1,8 +1,8 @@ package = "tidesdb" -version = "0.5.7-1" +version = "0.5.8-1" source = { url = "git://github.com/tidesdb/tidesdb-lua.git", - tag = "v0.5.7" + tag = "v0.5.8" } description = { summary = "Official Lua bindings for TidesDB - A high-performance embedded key-value storage engine",