MDEV-34358: Add debug status variables to verify encryption thread wait behavior

Added debug-only status variables Innodb_encryption_indefinite_waits
to track encryption indefinite thread wait instead of busy-waiting
when idle.

The counters are incremented in rotate_thread_t::wait_for_work()
and exposed via SHOW STATUS in debug builds only. Also added a
debug sync point 'rotate_only_2_timed_waits' to reduce the timed
wait threshold from 5 to 2 for faster testing of the indefinite
wait transition.

Author: Thirunarayanan

mysql-test/suite/encryption/r/busy_loop.result

@@ -0,0 +1,8 @@
+create table t1(a serial) engine=innoDB;
+set global innodb_encryption_threads=2;
+SET GLOBAL debug_dbug="+d,rotate_only_2_timed_waits";
+set global debug_key_management_version=10;
+SET GLOBAL innodb_encrypt_tables="FORCE";
+Indefinite waits happened
+drop table t1;
+# restart

mysql-test/suite/encryption/t/busy_loop.opt

@@ -0,0 +1,2 @@
+--innodb-tablespaces-encryption
+--plugin-load-add=$DEBUG_KEY_MANAGEMENT_SO

mysql-test/suite/encryption/t/busy_loop.test

@@ -0,0 +1,37 @@
+-- source include/have_innodb.inc
+-- source include/have_debug.inc
+if (`select count(*) = 0 from information_schema.plugins
+     where plugin_name = 'debug_key_management' and plugin_status='active'`)
+{
+  --skip Needs debug_key_management
+}
+create table t1(a serial) engine=innoDB;
+
+set global innodb_encryption_threads=2;
+SET GLOBAL debug_dbug="+d,rotate_only_2_timed_waits";
+set global debug_key_management_version=10;
+SET GLOBAL innodb_encrypt_tables="FORCE";
+
+--let $tables_count= `select count(*) + 1 + @@global.innodb_undo_tablespaces from information_schema.tables where engine = 'InnoDB'`
+
+let $wait_condition= select count(*) = $tables_count from information_schema.innodb_tablespaces_encryption where current_key_version=10;
+--source include/wait_condition.inc
+
+# Record initial indefinite wait count
+--let $initial_indefinite_waits= `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='INNODB_ENCRYPTION_INDEFINITE_WAITS'`
+
+sleep 20;
+
+# After sleep, verify threads used indefinite waits (not busy looping)
+--let $final_indefinite_waits= `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='INNODB_ENCRYPTION_INDEFINITE_WAITS'`
+
+# Ensure indefinite waits increased (threads are sleeping, not busy-looping)
+if (`SELECT $final_indefinite_waits <= $initial_indefinite_waits`)
+{
+  --echo ERROR: Encryption threads did not use indefinite waits during idle period
+  --echo This indicates potential busy-loop behavior
+  --die Encryption threads busy-looping detected
+}
+--echo Indefinite waits happened
+drop table t1;
+--source include/restart_mysqld.inc

mysql-test/suite/innodb/r/innodb_status_variables.result

@@ -3,7 +3,8 @@ WHERE variable_name LIKE 'INNODB_%'
 AND variable_name NOT IN
 ('INNODB_ADAPTIVE_HASH_HASH_SEARCHES','INNODB_ADAPTIVE_HASH_NON_HASH_SEARCHES',
 'INNODB_MEM_ADAPTIVE_HASH',
-'INNODB_BUFFERED_AIO_SUBMITTED','INNODB_BUFFER_POOL_PAGES_LATCHED');
+'INNODB_BUFFERED_AIO_SUBMITTED','INNODB_BUFFER_POOL_PAGES_LATCHED',
+'INNODB_ENCRYPTION_INDEFINITE_WAITS');
 variable_name
 INNODB_BACKGROUND_LOG_SYNC
 INNODB_BUFFER_POOL_DUMP_STATUS

mysql-test/suite/innodb/t/innodb_status_variables.test

@@ -4,4 +4,5 @@ WHERE variable_name LIKE 'INNODB_%'
 AND variable_name NOT IN
 ('INNODB_ADAPTIVE_HASH_HASH_SEARCHES','INNODB_ADAPTIVE_HASH_NON_HASH_SEARCHES',
  'INNODB_MEM_ADAPTIVE_HASH',
- 'INNODB_BUFFERED_AIO_SUBMITTED','INNODB_BUFFER_POOL_PAGES_LATCHED');
+ 'INNODB_BUFFERED_AIO_SUBMITTED','INNODB_BUFFER_POOL_PAGES_LATCHED',
+ 'INNODB_ENCRYPTION_INDEFINITE_WAITS');

storage/innobase/fil/fil0crypt.cc

@@ -64,6 +64,11 @@ uint srv_fil_crypt_rotate_key_age;
 /** Whether the encryption plugin does key rotation */
 Atomic_relaxed<bool> srv_encrypt_rotate;
 
+#ifdef UNIV_DEBUG
+/** Number of times encryption threads used indefinite wait */
+Atomic_counter<ulint> fil_crypt_indefinite_waits;
+#endif /* UNIV_DEBUG */
+
 /** Condition variable for srv_n_fil_crypt_threads_started */
 static pthread_cond_t fil_crypt_cond;
 
@@ -1131,8 +1136,10 @@ struct rotate_thread_t {
 
 		if (space == fil_system.space_list.end()) {
 			uint max_timed_waits = 5;
+			DBUG_EXECUTE_IF("rotate_only_2_timed_waits",
+					max_timed_waits = 2;);
 			if (timed_wait_count >= max_timed_waits) {
-				timed_wait_count= 0;
+				timed_wait_count = 0;
 				goto indefinite_wait;
 			}
 
@@ -1151,6 +1158,7 @@ struct rotate_thread_t {
 			}
 		} else {
 indefinite_wait:
+			ut_d(fil_crypt_indefinite_waits++;);
 			my_cond_wait(&fil_crypt_threads_cond,
 				     &fil_crypt_threads_mutex.m_mutex);
 		}

storage/innobase/handler/ha_innodb.cc

@@ -1106,6 +1106,10 @@ static SHOW_VAR innodb_status_variables[]= {
    &export_vars.innodb_n_temp_blocks_decrypted, SHOW_LONGLONG},
   {"encryption_num_key_requests", &export_vars.innodb_encryption_key_requests,
    SHOW_LONGLONG},
+#ifdef UNIV_DEBUG
+  {"encryption_indefinite_waits",
+   &fil_crypt_indefinite_waits, SHOW_SIZE_T},
+#endif /* UNIV_DEBUG */
 
   /* InnoDB bulk operations */
   {"bulk_operations", &export_vars.innodb_bulk_operations, SHOW_SIZE_T},

storage/innobase/include/fil0crypt.h

@@ -375,6 +375,11 @@ Return crypt statistics
 @param[out]	stat		Crypt statistics */
 void fil_crypt_total_stat(fil_crypt_stat_t *stat);
 
+#ifdef UNIV_DEBUG
+/** Number of times encryption threads used indefinite wait */
+extern Atomic_counter<ulint> fil_crypt_indefinite_waits;
+#endif /* UNIV_DEBUG */
+
 #include "fil0crypt.inl"
 #endif /* !UNIV_INNOCHECKSUM */