diff --git a/deps/xredis-gtid b/deps/xredis-gtid index c450ba710b5..52841a2297b 160000 --- a/deps/xredis-gtid +++ b/deps/xredis-gtid @@ -1 +1 @@ -Subproject commit c450ba710b56145e14c9952f28ad436049a93738 +Subproject commit 52841a2297b0efae5f0175a4dbd7c4b9539d3801 diff --git a/src/ctrip_swap.h b/src/ctrip_swap.h index cb12590884b..e720b74f101 100644 --- a/src/ctrip_swap.h +++ b/src/ctrip_swap.h @@ -1786,6 +1786,8 @@ int submitExpireClientRequest(client *c, robj *key, int force); /* Rocks */ #define ROCKS_DIR_MAX_LEN 512 #define ROCKS_DATA "data.rocks" +#define ROCKS_LOG "logs.rocks" + #define ROCKS_DISK_HEALTH_DETECT_FILE "disk_health_detect" /* Rocksdb engine */ diff --git a/src/ctrip_swap_rocks.c b/src/ctrip_swap_rocks.c index f3237bacece..13ff5f41bb3 100644 --- a/src/ctrip_swap_rocks.c +++ b/src/ctrip_swap_rocks.c @@ -255,6 +255,7 @@ static int rocksOpen(rocks *rocks) { rocksdb_options_set_compaction_filter_factory(rocks->cf_opts[META_CF], NULL); snprintf(dir, ROCKS_DIR_MAX_LEN, "%s/%d", ROCKS_DATA, rocks->rocksdb_epoch); + rocksdb_options_set_db_log_dir(rocks->db_opts, ROCKS_LOG); rocks->db = rocksdb_open_column_families(rocks->db_opts, dir, CF_COUNT, swap_cf_names, (const rocksdb_options_t *const *)rocks->cf_opts, rocks->cf_handles, errs); diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 218440ae4ac..3b5ad9c64b9 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -562,15 +562,35 @@ proc stop_bg_complex_data {handle} { catch {exec /bin/kill -9 $handle} } -proc populate {num prefix size} { - set rd [redis_deferring_client] - for {set j 0} {$j < $num} {incr j} { - $rd set $prefix$j [string repeat A $size] +# Write num keys with the given key prefix and value size (in bytes). If idx is +# given, it's the index (AKA level) used with the srv procedure and it specifies +# to which Redis instance to write the keys. +proc populate {num {prefix key:} {size 3} {idx 0} {prints false} {expires 0}} { + r $idx deferred 1 + if {$num > 16} {set pipeline 16} else {set pipeline $num} + set val [string repeat A $size] + for {set j 0} {$j < $pipeline} {incr j} { + if {$expires > 0} { + r $idx set $prefix$j $val ex $expires + } else { + r $idx set $prefix$j $val + } + if {$prints} {puts $j} + } + for {} {$j < $num} {incr j} { + if {$expires > 0} { + r $idx set $prefix$j $val ex $expires + } else { + r $idx set $prefix$j $val + } + r $idx read + if {$prints} {puts $j} } - for {set j 0} {$j < $num} {incr j} { - $rd read + for {set j 0} {$j < $pipeline} {incr j} { + r $idx read + if {$prints} {puts $j} } - $rd close + r $idx deferred 0 } proc get_child_pid {idx} { diff --git a/tests/unit/maxmemory.tcl b/tests/unit/maxmemory.tcl index 3298ef17904..509e4955def 100644 --- a/tests/unit/maxmemory.tcl +++ b/tests/unit/maxmemory.tcl @@ -191,11 +191,24 @@ proc test_slave_buffers {test_name cmd_count payload_len limit_memory pipeline} # send some 10mb worth of commands that don't increase the memory usage if {$pipeline == 1} { set rd_master [redis_deferring_client -1] + # Send commands in batches and read responses to avoid TCP deadlock. + # Without interleaving reads, the client's send buffer fills up when + # the server's output buffers are full (because we're not reading), + # causing flush to block indefinitely on slow machines. + set batch_size 10000 for {set k 0} {$k < $cmd_count} {incr k} { $rd_master setrange key:0 0 [string repeat A $payload_len] + if {($k + 1) % $batch_size == 0} { + # Drain responses to prevent TCP buffer deadlock + for {set j 0} {$j < $batch_size} {incr j} { + $rd_master read + } + } } - for {set k 0} {$k < $cmd_count} {incr k} { - #$rd_master read + # Read any remaining responses + set remaining [expr {$cmd_count % $batch_size}] + for {set k 0} {$k < $remaining} {incr k} { + $rd_master read } } else { for {set k 0} {$k < $cmd_count} {incr k} { diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index ae8207dd4a9..3830957af30 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -37,6 +37,13 @@ start_server {tags {"memefficiency" "memonly"}} { } run_solo {defrag} { + proc discard_replies_every {rd count frequency discard_num} { + if {$count % $frequency == 0} { + for {set k 0} {$k < $discard_num} {incr k} { + $rd read ; # Discard replies + } + } + } start_server {tags {"defrag" "memonly"} overrides {appendonly yes auto-aof-rewrite-percentage 0 save ""}} { if {[string match {*jemalloc*} [s mem_allocator]] && [r debug mallctl arenas.page] <= 8192} { test "Active defrag" { @@ -182,14 +189,21 @@ start_server {tags {"defrag" "memonly"} overrides {appendonly yes auto-aof-rewri # create big keys with 10k items set rd [redis_deferring_client] + set batch_size 100 for {set j 0} {$j < 10000} {incr j} { $rd hset bighash $j [concat "asdfasdfasdf" $j] $rd lpush biglist [concat "asdfasdfasdf" $j] $rd zadd bigzset $j [concat "asdfasdfasdf" $j] $rd sadd bigset [concat "asdfasdfasdf" $j] $rd xadd bigstream * item 1 value a + if {($j + 1) % $batch_size == 0} { + for {set i 0} {$i < [expr {$batch_size * 5}]} {incr i} { + $rd read + } + } } - for {set j 0} {$j < 50000} {incr j} { + set remaining [expr {(10000 % $batch_size) * 5}] + for {set j 0} {$j < $remaining} {incr j} { $rd read ; # Discard replies } @@ -207,22 +221,20 @@ start_server {tags {"defrag" "memonly"} overrides {appendonly yes auto-aof-rewri } # add a mass of string keys + set count 0 for {set j 0} {$j < 500000} {incr j} { $rd setrange $j 150 a + incr count + discard_replies_every $rd $count 10000 10000 } - for {set j 0} {$j < 500000} {incr j} { - $rd read ; # Discard replies - } - assert_equal [r dbsize] 500010 - + set count 0 # create some fragmentation for {set j 0} {$j < 500000} {incr j 2} { $rd del $j + incr count + discard_replies_every $rd $count 10000 10000 } - for {set j 0} {$j < 500000} {incr j 2} { - $rd read ; # Discard replies - } - assert_equal [r dbsize] 250010 + # start defrag after 120 ;# serverCron only updates the info once in 100ms @@ -307,13 +319,12 @@ start_server {tags {"defrag" "memonly"} overrides {appendonly yes auto-aof-rewri # add a mass of list nodes to two lists (allocations are interlaced) set val [string repeat A 100] ;# 5 items of 100 bytes puts us in the 640 bytes bin, which has 32 regs, so high potential for fragmentation set elements 500000 + set count 0 for {set j 0} {$j < $elements} {incr j} { $rd lpush biglist1 $val $rd lpush biglist2 $val - } - for {set j 0} {$j < $elements} {incr j} { - $rd read ; # Discard replies - $rd read ; # Discard replies + incr count + discard_replies_every $rd $count 1000 2000 } # create some fragmentation @@ -418,11 +429,11 @@ start_server {tags {"defrag" "memonly"} overrides {appendonly yes auto-aof-rewri # add a mass of keys with 600 bytes values, fill the bin of 640 bytes which has 32 regs per slab. set rd [redis_deferring_client] set keys 640000 + set count 0 for {set j 0} {$j < $keys} {incr j} { $rd setrange $j 600 x - } - for {set j 0} {$j < $keys} {incr j} { - $rd read ; # Discard replies + incr count + discard_replies_every $rd $count 10000 10000 } # create some fragmentation of 50% @@ -431,9 +442,7 @@ start_server {tags {"defrag" "memonly"} overrides {appendonly yes auto-aof-rewri $rd del $j incr sent incr j 1 - } - for {set j 0} {$j < $sent} {incr j} { - $rd read ; # Discard replies + discard_replies_every $rd $sent 10000 10000 } # create higher fragmentation in the first slab