Improved the reliability of cluster replica sync tests (#9628)

Improved the reliability of cluster replica sync tests

src/config.c

@@ -2694,6 +2694,7 @@ standardConfig configs[] = {
     /* Other configs */
     createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */
     createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG, NULL, NULL),
+    createOffTConfig("loading-process-events-interval-bytes", NULL, MODIFIABLE_CONFIG, 1024, INT_MAX, server.loading_process_events_interval_bytes, 1024*1024*2, INTEGER_CONFIG, NULL, NULL),
 
 #ifdef USE_OPENSSL
     createIntConfig("tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */

src/server.c

@@ -3198,7 +3198,6 @@ void initServerConfig(void) {
     server.cluster_module_flags = CLUSTER_MODULE_FLAG_NONE;
     server.migrate_cached_sockets = dictCreate(&migrateCacheDictType);
     server.next_client_id = 1; /* Client IDs, start from 1 .*/
-    server.loading_process_events_interval_bytes = (1024*1024*2);
     server.page_size = sysconf(_SC_PAGESIZE);
     server.pause_cron = 0;
 

tests/cluster/tests/22-replica-in-sync.tcl

@@ -27,26 +27,25 @@ proc is_replica_online {info_repl} {
 
 set master_id 0
 
-test "Fill up" {
-    R $master_id debug populate 10000000 key 100
+test "Fill up primary with data" {
+    # Set 1 MB of data
+    R $master_id debug populate 1000 key 1000
 }
 
 test "Add new node as replica" {
-    set replica_id [cluster_find_available_slave 1]
-    set master_myself [get_myself $master_id]
-    set replica_myself [get_myself $replica_id]
-    set replica [dict get $replica_myself id]
-    R $replica_id cluster replicate [dict get $master_myself id]
+    set replica_id 1
+    set replica [R $replica_id CLUSTER MYID]
+    R $replica_id cluster replicate [R $master_id CLUSTER MYID]
 }
 
 test "Check digest and replica state" {
-    R 1 readonly
     wait_for_condition 1000 50 {
         [is_in_slots $master_id $replica]
     } else {
         fail "New replica didn't appear in the slots"
     }
-    wait_for_condition 1000 50 {
+
+    wait_for_condition 100 50 {
         [is_replica_online [R $master_id info replication]]
     } else {
         fail "Replica is down for too long"
@@ -57,10 +56,20 @@ test "Check digest and replica state" {
 
 test "Replica in loading state is hidden" {
     # Kill replica client for master and load new data to the primary
-    R $master_id multi
     R $master_id config set repl-backlog-size 100
+
+    # Set the key load delay so that it will take at least
+    # 2 seconds to fully load the data.
+    R $replica_id config set key-load-delay 4000
+
+    # Trigger event loop processing every 1024 bytes, this trigger
+    # allows us to send and receive cluster messages, so we are setting
+    # it low so that the cluster messages are sent more frequently.
+    R $replica_id config set loading-process-events-interval-bytes 1024
+
+    R $master_id multi
     R $master_id client kill type replica
-    set num 10000
+    set num 100
     set value [string repeat A 1024]
     for {set j 0} {$j < $num} {incr j} {
         set key "{0}"
@@ -69,28 +78,34 @@ test "Replica in loading state is hidden" {
     }
     R $master_id exec
 
-    # Check that replica started loading
-    wait_for_condition 1000 50 {
-        [s $replica_id loading] eq 1
+    # The master will be the last to know the replica
+    # is loading, so we will wait on that and assert
+    # the replica is loading afterwards. 
+    wait_for_condition 100 50 {
+        ![is_in_slots $master_id $replica]
     } else {
-        fail "Replica didn't enter loading state"
+        fail "Replica was always present in cluster slots"
     }
-    # Check that replica is not in cluster slots
-    assert {![is_in_slots $master_id $replica]}
+    assert_equal 1 [s $replica_id loading]
 
-    # Wait for sync to finish
-    wait_for_condition 1000 50 {
-        [s $replica_id loading] eq 0
+    # Wait for the replica to finish full-sync and become online
+    wait_for_condition 200 50 {
+        [s $replica_id master_link_status] eq "up"
     } else {
-        fail "Replica is in loading state for too long"
+        fail "Replica didn't finish loading"
     }
 
-    # Check replica is back to cluster slots
-    wait_for_condition 1000 50 {
+    # Return configs to default values
+    R $replica_id config set loading-process-events-interval-bytes 2097152
+    R $replica_id config set key-load-delay 0
+
+    # Check replica is back in cluster slots
+    wait_for_condition 100 50 {
         [is_in_slots $master_id $replica] 
     } else {
         fail "Replica is not back to slots"
     }
+    assert_equal 1 [is_in_slots $replica_id $replica] 
 }
 
 test "Check disconnected replica not hidden from slots" {

tests/cluster/tests/includes/init-tests.tcl

@@ -39,6 +39,8 @@ test "Cluster nodes hard reset" {
         R $id EXEC
         R $id config set cluster-node-timeout $node_timeout
         R $id config set cluster-slave-validity-factor 10
+        R $id config set loading-process-events-interval-bytes 2097152
+        R $id config set key-load-delay 0
         R $id config rewrite
     }
 }