From 2cb1d1c5975fe5fc151dfae72a8fba3d9d2a8bb9 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Thu, 16 Jul 2026 14:42:53 +0000 Subject: [PATCH 1/3] DAOS-19174 test: dmg system rebuild stop/start Add variant to rebuild/interactive.py for - dmg system rebuild stop - dmg system rebuild start Run with multiple pools. Run with background IOR. Test-tag: RbldInteractive Test-repeat: 10 skip-unit-tests: True skip-fault-injection-test: True Adjust Ior.run() to re-use params if already set. Signed-off-by: Dalton Bohning --- src/tests/ftest/rebuild/interactive.py | 360 +++++++++++++++++------ src/tests/ftest/rebuild/interactive.yaml | 15 +- src/tests/ftest/util/ior_utils.py | 36 ++- 3 files changed, 308 insertions(+), 103 deletions(-) diff --git a/src/tests/ftest/rebuild/interactive.py b/src/tests/ftest/rebuild/interactive.py index 43f3ce0e74c..1e6734b03f1 100644 --- a/src/tests/ftest/rebuild/interactive.py +++ b/src/tests/ftest/rebuild/interactive.py @@ -3,13 +3,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import threading import time +from collections import defaultdict from functools import partial +from multiprocessing import Queue from apricot import TestWithServers from data_utils import assert_val_in_list from exception_utils import CommandFailure -from ior_utils import get_ior +from ior_utils import get_ior, thread_run_ior from job_manager_utils import get_job_manager @@ -19,6 +22,81 @@ class RbldInteractive(TestWithServers): :avocado: recursive """ + REBUILD_STOP_MAX_WAIT_TIME = 30 + REBUILD_STOP_SLEEP = 3 + + def _pool_rebuild_stop(self, pool): + """Stop rebuild with dmg pool rebuild stop. + + Args: + pool (TestPool): pool to stop rebuild on + + Returns: + CmdResult: Object that contains exit status, stdout, and other information. + """ + time_start = time.time() + while True: + try: + return pool.rebuild_stop() + except CommandFailure as error: + # Any error other than DER_NONEXIST is a real error + if 'DER_NONEXIST' not in str(error): + raise + # If we exceed the max wait time, fail the test + if time.time() - time_start > self.REBUILD_STOP_MAX_WAIT_TIME: + self.fail( + f'Failed to stop rebuild after {self.REBUILD_STOP_MAX_WAIT_TIME} seconds') + # Otherwise, sleep and retry + self.log.info( + 'Assuming rebuild is not started yet. Retrying in %s seconds...', + self.REBUILD_STOP_SLEEP) + time.sleep(self.REBUILD_STOP_SLEEP) + + def _system_rebuild_stop(self, dmg): + """Stop rebuild with dmg system rebuild stop. + + Args: + dmg (DmgCommand): DmgCommand object to use for stopping rebuild + + Returns: + CmdResult: Object that contains exit status, stdout, and other information. + """ + rebuild_stopped = defaultdict(lambda: False) + time_start = time.time() + while True: + with dmg.no_exception(): + result = dmg.system_rebuild_stop() + + # If the command did not error, all is good + if result['status'] == 0: + return result + + # The command errored, but anything other than DER_NONEXIST is a real error + if 'DER_NONEXIST' not in result['error']: + raise CommandFailure( + f'Unexpected error stopping rebuild: {result["error"]}') + + # The command failed with DER_NONEXIST, + # so keep a running check of which pools have stopped rebuild + for pool_result in result['response']['results']: + rebuild_stopped[pool_result['id']] |= pool_result['errored'] is False + + # If all pools have stopped rebuild, all is good + if all(rebuild_stopped.values()): + return result + + # If we exceed the max wait time, fail the test + if time.time() - time_start > self.REBUILD_STOP_MAX_WAIT_TIME: + self.fail( + f'Failed to stop rebuild after ' + f'{self.REBUILD_STOP_MAX_WAIT_TIME} seconds') + + # Otherwise, sleep and retry + self.log.info( + 'Assuming rebuild is not started yet. Retrying in %s seconds...', + self.REBUILD_STOP_SLEEP) + time.sleep(self.REBUILD_STOP_SLEEP) + def test_rebuild_interactive(self): """ Use Cases: @@ -29,8 +107,8 @@ def test_rebuild_interactive(self): :avocado: tags=rebuild,pool :avocado: tags=RbldInteractive,test_rebuild_interactive """ - self.log_step("Setup pool") - pool = self.get_pool(connect=False) + self.log_step("Setup first pool") + pool1 = self.get_pool(connect=False) # Collect server configuration information server_count = len(self.hostlist_servers) @@ -41,38 +119,119 @@ def test_rebuild_interactive(self): server_count, engines_per_host, targets_per_engine) self.log_step('Create container and run IOR') - cont_ior = self.get_container(pool, namespace='/run/cont_ior/*') + cont1 = self.get_container(pool1) ior_flags_write = self.params.get('flags_write', '/run/ior/*') + ior_flags_read = self.params.get('flags_read', '/run/ior/*') ior_ppn = self.params.get('ppn', '/run/ior/*') - job_manager = get_job_manager(self, subprocess=False) - ior = get_ior( - self, job_manager, self.hostlist_clients, self.workdir, None, namespace='/run/ior/*') - ior.manager.job.update_params(flags=ior_flags_write, dfs_oclass=cont_ior.oclass.value) - ior.run(cont_ior.pool, cont_ior, None, ior_ppn, display_space=False) - + ior1 = get_ior( + self, get_job_manager(self, subprocess=False), self.hostlist_clients, + self.workdir, None, namespace='/run/ior/*') + ior1.manager.job.update_params( + flags=ior_flags_write, dfs_oclass=cont1.oclass.value, + dfs_pool=pool1.identifier, dfs_cont=cont1.identifier) + ior1.run(ppn=ior_ppn, display_space=False) + + # Update ior with read flags for verification later + ior1.manager.job.update_params(flags=ior_flags_read) + + # Launch background IOR + cont_background = self.get_container(pool1) + thread_queue = Queue() + ior_background_namespace = "/run/ior_background/*" + ior_kwargs = { + "thread_queue": thread_queue, + "job_id": 0, + "test": self, + "manager": get_job_manager(self, subprocess=False), + "log": "ior_thread.log", + "hosts": self.hostlist_clients, + "path": self.workdir, + "slots": None, + "pool": pool1, + "container": cont_background, + "processes": self.params.get("np", ior_background_namespace), + "ppn": self.params.get("ppn", ior_background_namespace), + "display_space": False, + "namespace": ior_background_namespace, + "ior_params": { + "dfs_oclass": cont_background.oclass.value + } + } + ior_thread = threading.Thread(target=thread_run_ior, kwargs=ior_kwargs) + ior_thread.start() + if not ior_thread.is_alive(): + self.fail("Background IOR thread failed to start") + + rebuild_sequence_start = time.time() self.__run_rebuild_interactive( - pool, cont_ior, ior, + [pool1], [ior1], num_ranks_to_exclude=1, exclude_method='dmg pool exclude', - reint_method='dmg pool reintegrate') + reint_method='dmg pool reintegrate', + stop_method='dmg pool rebuild stop', + start_method='dmg pool rebuild start') + rebuild_sequence_duration = time.time() - rebuild_sequence_start + self.log.info("Rebuild sequence completed in %.2f seconds", rebuild_sequence_duration) + + self.log.info("Waiting for background IOR to finish") + ior_thread.join() + if thread_queue.empty(): + self.fail("Did not receive a result from background IOR") + ior_result = thread_queue.get() + self.log.debug("Result from background IOR:") + for name in ("command", "exit_status", "interrupted", "duration"): + self.log.debug(" %s: %s", name, getattr(ior_result["result"], name)) + for name in ("stdout", "stderr"): + self.log.debug(" %s:", name) + for line in getattr(ior_result["result"], name).splitlines(): + self.log.debug(" %s", line) + if ior_result["result"].exit_status != 0: + self.fail("Background IOR failed") + ior_thread_duration = ior_result["result"].duration + self.log.info("Background IOR completed in %.2f seconds", ior_thread_duration) + if ior_thread_duration < rebuild_sequence_duration: + self.fail( + "Background IOR completed before rebuild sequence. " + "Need to increase background IOR runtime or iterations.") + + self.log_step("Setup second pool") + pool2 = self.get_pool(connect=False) + + self.log_step('Create second container and run IOR') + cont2 = self.get_container(pool2) + + ior2 = get_ior( + self, get_job_manager(self, subprocess=False), self.hostlist_clients, + self.workdir, None, namespace='/run/ior/*') + ior2.manager.job.update_params( + flags=ior_flags_write, dfs_oclass=cont2.oclass.value, + dfs_pool=pool2.identifier, dfs_cont=cont2.identifier) + ior2.run(ppn=ior_ppn, display_space=False) + + # Update ior with read flags for verification later + ior2.manager.job.update_params(flags=ior_flags_read) self.__run_rebuild_interactive( - pool, cont_ior, ior, + [pool1, pool2], [ior1, ior2], num_ranks_to_exclude=1, exclude_method='dmg system exclude', - reint_method='dmg system reintegrate') + reint_method='dmg system reintegrate', + stop_method='dmg system rebuild stop', + start_method='dmg system rebuild start') self.log_step('Test Passed') - def __run_rebuild_interactive(self, pool, cont_ior, ior, - num_ranks_to_exclude, exclude_method, reint_method): + def __run_rebuild_interactive(self, pools, iors, + num_ranks_to_exclude, exclude_method, reint_method, + stop_method, start_method): + # pylint: disable=too-many-branches """Run interactive rebuild test sequence. + Args: - pool (TestPool): pool to use - cont_ior (TestContainer): container used for IOR - iort (Ior): the Ior object + pools (list): list of TestPool to use + iors (list): list of Ior objects to verify data consistency num_ranks_to_exclude (int): number of ranks to exclude/reintegrate exclude_method (str): method to exclude ranks. Must be in - 'dmg pool exclude' @@ -80,125 +239,152 @@ def __run_rebuild_interactive(self, pool, cont_ior, ior, reint_method (str): method to reintegrate ranks. Must be in - 'dmg pool reintegrate' - 'dmg system reintegrate' + stop_method (str): method to stop rebuild with. Must be in + - 'dmg pool rebuild stop' + - 'dmg system rebuild stop' + start_method (str): method to start rebuild with. Must be in + - 'dmg pool rebuild start' + - 'dmg system rebuild start' """ - - ior_flags_read = self.params.get('flags_read', '/run/ior/*') - ior_ppn = self.params.get('ppn', '/run/ior/*') + dmg = self.get_dmg_command() self.log_step('Verify pool state before rebuild') - self.__verify_pool_query( - pool, rebuild_status=0, rebuild_state=['idle', 'done'], disabled_ranks=[]) + for pool in pools: + self.__verify_pool_query( + pool, rebuild_status=0, rebuild_state=['idle', 'done'], disabled_ranks=[]) ranks_to_exclude = self.random.sample( list(self.server_managers[0].ranks.keys()), k=num_ranks_to_exclude) - self.log_step(f'Exclude random rank {ranks_to_exclude}') + self.log_step(f'{exclude_method} - Exclude random rank {ranks_to_exclude}') if exclude_method == 'dmg pool exclude': - pool.exclude(ranks_to_exclude) + for pool in pools: + pool.exclude(ranks_to_exclude) elif exclude_method == 'dmg system exclude': - pool.dmg.system_exclude(ranks_to_exclude) + dmg.system_exclude(ranks_to_exclude) else: self.fail(f'Unsupported exclude_method: {exclude_method}') self.log_step(f'{exclude_method} - Wait for rebuild to start') - pool.wait_for_rebuild_to_start(interval=1) - - self.log_step(f'{exclude_method} - Manually stop rebuild') - for i in range(4): - try: - pool.rebuild_stop() - break - except CommandFailure as error: - if i == 3 or 'DER_NONEXIST' not in str(error): - raise - self.log.info('Assuming rebuild is not started yet. Retrying in 3 seconds...') - time.sleep(3) + for pool in pools: + pool.wait_for_rebuild_to_start(interval=1) + + self.log_step(f'{exclude_method} - Manually stop rebuild with {stop_method}') + if stop_method == 'dmg pool rebuild stop': + for pool in pools: + self._pool_rebuild_stop(pool) + elif stop_method == 'dmg system rebuild stop': + self._system_rebuild_stop(dmg) + else: + self.fail(f'Unsupported stop_method: {stop_method}') self.log_step(f'{exclude_method} - Wait for rebuild to stop') - pool.wait_for_rebuild_to_stop(interval=3) + for pool in pools: + pool.wait_for_rebuild_to_stop(interval=3) self.log_step(f'{exclude_method} - Verify pool state after rebuild stopped') - self.__verify_pool_query( - pool, rebuild_status=-2027, rebuild_state=['idle'], - disabled_ranks=ranks_to_exclude) + for pool in pools: + self.__verify_pool_query( + pool, rebuild_status=-2027, rebuild_state=['idle'], + disabled_ranks=ranks_to_exclude) self.log_step(f'{exclude_method} - Verify IOR after rebuild stopped') - ior.manager.job.update_params(flags=ior_flags_read) - ior.run(cont_ior.pool, cont_ior, None, ior_ppn, display_space=False) - - self.log_step(f'{exclude_method} - Manually start rebuild') - pool.rebuild_start() + for ior in iors: + ior.run(display_space=False) + + self.log_step(f'{exclude_method} - Manually start rebuild with {start_method}') + if start_method == 'dmg pool rebuild start': + for pool in pools: + pool.rebuild_start() + elif start_method == 'dmg system rebuild start': + dmg.system_rebuild_start() + else: + self.fail(f'Unsupported start_method: {start_method}') self.log_step(f'{exclude_method} - Wait for rebuild to start') - pool.wait_for_rebuild_to_start(interval=1) + for pool in pools: + pool.wait_for_rebuild_to_start(interval=1) self.log_step(f'{exclude_method} - Wait for rebuild to end') - pool.wait_for_rebuild_to_end(interval=3) + for pool in pools: + pool.wait_for_rebuild_to_end(interval=3) self.log_step(f'{exclude_method} - Verify pool state after rebuild completed') - self.__verify_pool_query( - pool, rebuild_status=0, rebuild_state=['idle', 'done'], - disabled_ranks=ranks_to_exclude) + for pool in pools: + self.__verify_pool_query( + pool, rebuild_status=0, rebuild_state=['idle', 'done'], + disabled_ranks=ranks_to_exclude) self.log_step(f'{exclude_method} - Verify IOR after rebuild completed') - ior.manager.job.update_params(flags=ior_flags_read) - ior.run(cont_ior.pool, cont_ior, None, ior_ppn, display_space=False) + for ior in iors: + ior.run(display_space=False) if exclude_method == 'dmg system exclude': self.log_step(f'{exclude_method} - Clear exclusion of ranks') - pool.dmg.system_clear_exclude(ranks_to_exclude) + dmg.system_clear_exclude(ranks_to_exclude) self.log_step(f'{exclude_method} - Start previously admin-excluded ranks') - pool.dmg.system_start(ranks_to_exclude) + dmg.system_start(ranks_to_exclude) - self.log_step('Reintegrate excluded ranks') + self.log_step(f'{reint_method} - Reintegrate excluded ranks') if reint_method == 'dmg pool reintegrate': - pool.reintegrate(ranks_to_exclude) + for pool in pools: + pool.reintegrate(ranks_to_exclude) elif reint_method == 'dmg system reintegrate': - pool.dmg.system_reintegrate(ranks_to_exclude) + dmg.system_reintegrate(ranks_to_exclude) else: self.fail(f'Unsupported reint_method: {reint_method}') self.log_step(f'{reint_method} - Wait for rebuild to start') - pool.wait_for_rebuild_to_start(interval=1) - - self.log_step(f'{reint_method} - Manually stop rebuild') - for i in range(4): - try: - pool.rebuild_stop() - break - except CommandFailure as error: - if i == 3 or 'DER_NONEXIST' not in str(error): - raise - self.log.info('Assuming rebuild is not started yet. Retrying in 3 seconds...') - time.sleep(3) + for pool in pools: + pool.wait_for_rebuild_to_start(interval=1) + + self.log_step(f'{reint_method} - Manually stop rebuild with {stop_method}') + if stop_method == 'dmg pool rebuild stop': + for pool in pools: + self._pool_rebuild_stop(pool) + elif stop_method == 'dmg system rebuild stop': + self._system_rebuild_stop(dmg) + else: + self.fail(f'Unsupported stop_method: {stop_method}') self.log_step(f'{reint_method} - Wait for rebuild to stop') - pool.wait_for_rebuild_to_stop(interval=3) + for pool in pools: + pool.wait_for_rebuild_to_stop(interval=3) self.log_step(f'{reint_method} - Verify pool state after rebuild stopped') - self.__verify_pool_query( - pool, rebuild_status=-2027, rebuild_state=['idle'], - disabled_ranks=[]) + for pool in pools: + self.__verify_pool_query( + pool, rebuild_status=-2027, rebuild_state=['idle'], + disabled_ranks=[]) self.log_step(f'{reint_method} - Verify IOR after rebuild stopped') - ior.manager.job.update_params(flags=ior_flags_read) - ior.run(cont_ior.pool, cont_ior, None, ior_ppn, display_space=False) - - self.log_step(f'{reint_method} - Manually start rebuild') - pool.rebuild_start() + for ior in iors: + ior.run(display_space=False) + + self.log_step(f'{reint_method} - Manually start rebuild with {start_method}') + if start_method == 'dmg pool rebuild start': + for pool in pools: + pool.rebuild_start() + elif start_method == 'dmg system rebuild start': + dmg.system_rebuild_start() + else: + self.fail(f'Unsupported start_method: {start_method}') self.log_step(f'{reint_method} - Wait for rebuild to start') - pool.wait_for_rebuild_to_start(interval=1) + for pool in pools: + pool.wait_for_rebuild_to_start(interval=1) self.log_step(f'{reint_method} - Wait for rebuild to end') - pool.wait_for_rebuild_to_end(interval=3) + for pool in pools: + pool.wait_for_rebuild_to_end(interval=3) self.log_step(f'{reint_method} - Verify pool state after rebuild completed') - self.__verify_pool_query( - pool, rebuild_status=0, rebuild_state=['idle', 'done'], disabled_ranks=[]) + for pool in pools: + self.__verify_pool_query( + pool, rebuild_status=0, rebuild_state=['idle', 'done'], disabled_ranks=[]) self.log_step(f'{reint_method} - Verify IOR after rebuild completed') - ior.manager.job.update_params(flags=ior_flags_read) - ior.run(cont_ior.pool, cont_ior, None, ior_ppn, display_space=False) + for ior in iors: + ior.run(display_space=False) def __verify_pool_query(self, pool, rebuild_status, rebuild_state, disabled_ranks): """Verify pool query. diff --git a/src/tests/ftest/rebuild/interactive.yaml b/src/tests/ftest/rebuild/interactive.yaml index 768557cd80a..4aa56e0fadc 100644 --- a/src/tests/ftest/rebuild/interactive.yaml +++ b/src/tests/ftest/rebuild/interactive.yaml @@ -2,7 +2,7 @@ hosts: test_servers: 7 test_clients: 1 -timeout: 500 +timeout: 560 server_config: name: daos_server @@ -33,7 +33,7 @@ pool: size: 50% pool_query_timeout: 30 -cont_ior: +container: type: POSIX properties: rd_fac:3 oclass: EC_8P3GX @@ -46,3 +46,14 @@ ior: block_size: 128M flags_write: "-v -w -k -G 1 -F" flags_read: "-v -r -R -k -G 1 -F" + +ior_background: + np: 1 + test_file: /testFile + api: DFS + transfer_size: 1M + flags: "-v -w -r -R -k -G 1 -F" + block_size: '150G' + repetitions: 4 # 4 iterations of write + read for 30s each, + sw_deadline: 30 # for a total of ~240s + sw_wearout: 1 diff --git a/src/tests/ftest/util/ior_utils.py b/src/tests/ftest/util/ior_utils.py index ea0d1a02516..ff11a375259 100644 --- a/src/tests/ftest/util/ior_utils.py +++ b/src/tests/ftest/util/ior_utils.py @@ -85,8 +85,9 @@ def run_ior(test, manager, log, hosts, path, slots, pool, container, processes, def thread_run_ior(thread_queue, job_id, test, manager, log, hosts, path, slots, - pool, container, processes, ppn, intercept, plugin_path, dfuse, - display_space, fail_on_warning, namespace, ior_params): + pool, container, processes, ppn=None, intercept=None, plugin_path=None, + dfuse=None, display_space=True, fail_on_warning=False, namespace="/run/ior/*", + ior_params=None): # pylint: disable=too-many-arguments """Start an IOR thread with thread queue for failure analysis. @@ -572,17 +573,21 @@ def get_unique_log(self, container): parts.append('read') return '.'.join(['_'.join(parts), 'log']) - def run(self, pool, container, processes, ppn=None, intercept=None, plugin_path=None, - dfuse=None, display_space=True, fail_on_warning=False, unique_log=True, il_report=None): + def run(self, pool=None, container=None, processes=None, ppn=None, intercept=None, + plugin_path=None, dfuse=None, display_space=True, fail_on_warning=False, + unique_log=True, il_report=None): # pylint: disable=too-many-arguments """Run ior. Args: - pool (TestPool): DAOS test pool object - container (TestContainer): DAOS test container object. - processes (int): number of processes to run - ppn (int, optional): number of processes per node to run. If specified it will override - the processes input. Defaults to None. + pool (TestPool, optional): DAOS test pool object. Overrides the current pool. + Defaults to None. + container (TestContainer, optional): DAOS test container object. + Overrides the current container. Defaults to None. + processes (int, optional): number of processes to run. Overrides the current processes. + Defaults to None. + ppn (int, optional): number of processes per node to run. Overrides the current ppn. + Takes precedent over `processes`. Defaults to None. intercept (str, optional): path to interception library. Defaults to None. plugin_path (str, optional): HDF5 vol connector library path. This will enable dfuse working directory which is needed to run vol connector for DAOS. Default is None. @@ -606,7 +611,10 @@ def run(self, pool, container, processes, ppn=None, intercept=None, plugin_path= result = None error_message = None - self.command.set_daos_params(pool, container.identifier) + if pool: + self.command.update_params(dfs_pool=pool.identifier) + if container: + self.command.update_params(dfs_cont=container.identifier) if intercept: self.env["LD_PRELOAD"] = intercept @@ -635,7 +643,7 @@ def run(self, pool, container, processes, ppn=None, intercept=None, plugin_path= # Pass only processes or ppn to be compatible with previous behavior if ppn is not None: self.manager.assign_processes(ppn=ppn) - else: + elif processes is not None: self.manager.assign_processes(processes=processes) self.manager.assign_environment(self.env) @@ -643,11 +651,11 @@ def run(self, pool, container, processes, ppn=None, intercept=None, plugin_path= if fail_on_warning and "WARNING" not in self.manager.check_results_list: self.manager.check_results_list.append("WARNING") - if unique_log: + if container and unique_log: self.update_log_file(self.get_unique_log(container)) try: - if display_space: + if pool and display_space: pool.display_space() result = self.manager.run() @@ -655,7 +663,7 @@ def run(self, pool, container, processes, ppn=None, intercept=None, plugin_path= error_message = "IOR Failed:\n {}".format("\n ".join(str(error).split("\n"))) finally: - if not self.manager.run_as_subprocess and display_space: + if not self.manager.run_as_subprocess and pool and display_space: pool.display_space() if error_message: From 4c8fd75ed7ec2c538445b42978acf2a847b45212 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Tue, 4 Aug 2026 17:32:29 +0000 Subject: [PATCH 2/3] double IOR data Test-tag: RbldInteractive Test-repeat: 10 skip-unit-tests: True skip-fault-injection-test: True Signed-off-by: Dalton Bohning --- src/tests/ftest/rebuild/interactive.py | 10 +++++----- src/tests/ftest/rebuild/interactive.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tests/ftest/rebuild/interactive.py b/src/tests/ftest/rebuild/interactive.py index 1e6734b03f1..9031d1c0941 100644 --- a/src/tests/ftest/rebuild/interactive.py +++ b/src/tests/ftest/rebuild/interactive.py @@ -23,7 +23,7 @@ class RbldInteractive(TestWithServers): """ REBUILD_STOP_MAX_WAIT_TIME = 30 - REBUILD_STOP_SLEEP = 3 + REBUILD_STOP_SLEEP_TIME = 3 def _pool_rebuild_stop(self, pool): """Stop rebuild with dmg pool rebuild stop. @@ -49,8 +49,8 @@ def _pool_rebuild_stop(self, pool): # Otherwise, sleep and retry self.log.info( 'Assuming rebuild is not started yet. Retrying in %s seconds...', - self.REBUILD_STOP_SLEEP) - time.sleep(self.REBUILD_STOP_SLEEP) + self.REBUILD_STOP_SLEEP_TIME) + time.sleep(self.REBUILD_STOP_SLEEP_TIME) def _system_rebuild_stop(self, dmg): """Stop rebuild with dmg system rebuild stop. @@ -94,8 +94,8 @@ def _system_rebuild_stop(self, dmg): # Otherwise, sleep and retry self.log.info( 'Assuming rebuild is not started yet. Retrying in %s seconds...', - self.REBUILD_STOP_SLEEP) - time.sleep(self.REBUILD_STOP_SLEEP) + self.REBUILD_STOP_SLEEP_TIME) + time.sleep(self.REBUILD_STOP_SLEEP_TIME) def test_rebuild_interactive(self): """ diff --git a/src/tests/ftest/rebuild/interactive.yaml b/src/tests/ftest/rebuild/interactive.yaml index 4aa56e0fadc..51cb8c82a2f 100644 --- a/src/tests/ftest/rebuild/interactive.yaml +++ b/src/tests/ftest/rebuild/interactive.yaml @@ -43,7 +43,7 @@ ior: test_file: /testFile api: DFS transfer_size: 1M - block_size: 128M + block_size: 256M flags_write: "-v -w -k -G 1 -F" flags_read: "-v -r -R -k -G 1 -F" From 7a427b14fcc4df1066c187b8028da0ac35455e58 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Wed, 5 Aug 2026 13:36:20 +0000 Subject: [PATCH 3/3] do not wait for rebuild to start before stopping The stop will keep looping and retrying Test-tag: RbldInteractive Test-repeat: 10 skip-unit-tests: True skip-fault-injection-test: True Signed-off-by: Dalton Bohning --- src/tests/ftest/rebuild/interactive.py | 16 ++++++++-------- src/tests/ftest/rebuild/interactive.yaml | 2 +- src/tests/ftest/util/ior_utils.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/tests/ftest/rebuild/interactive.py b/src/tests/ftest/rebuild/interactive.py index 9031d1c0941..dc0708518b3 100644 --- a/src/tests/ftest/rebuild/interactive.py +++ b/src/tests/ftest/rebuild/interactive.py @@ -135,7 +135,7 @@ def test_rebuild_interactive(self): # Update ior with read flags for verification later ior1.manager.job.update_params(flags=ior_flags_read) - # Launch background IOR + self.log_step('Start IOR in the background') cont_background = self.get_container(pool1) thread_queue = Queue() ior_background_namespace = "/run/ior_background/*" @@ -174,7 +174,7 @@ def test_rebuild_interactive(self): rebuild_sequence_duration = time.time() - rebuild_sequence_start self.log.info("Rebuild sequence completed in %.2f seconds", rebuild_sequence_duration) - self.log.info("Waiting for background IOR to finish") + self.log_step("Wait for background IOR to finish") ior_thread.join() if thread_queue.empty(): self.fail("Did not receive a result from background IOR") @@ -264,9 +264,9 @@ def __run_rebuild_interactive(self, pools, iors, else: self.fail(f'Unsupported exclude_method: {exclude_method}') - self.log_step(f'{exclude_method} - Wait for rebuild to start') - for pool in pools: - pool.wait_for_rebuild_to_start(interval=1) + # self.log_step(f'{exclude_method} - Wait for rebuild to start') + # for pool in pools: + # pool.wait_for_rebuild_to_start(interval=1) self.log_step(f'{exclude_method} - Manually stop rebuild with {stop_method}') if stop_method == 'dmg pool rebuild stop': @@ -333,9 +333,9 @@ def __run_rebuild_interactive(self, pools, iors, else: self.fail(f'Unsupported reint_method: {reint_method}') - self.log_step(f'{reint_method} - Wait for rebuild to start') - for pool in pools: - pool.wait_for_rebuild_to_start(interval=1) + # self.log_step(f'{reint_method} - Wait for rebuild to start') + # for pool in pools: + # pool.wait_for_rebuild_to_start(interval=1) self.log_step(f'{reint_method} - Manually stop rebuild with {stop_method}') if stop_method == 'dmg pool rebuild stop': diff --git a/src/tests/ftest/rebuild/interactive.yaml b/src/tests/ftest/rebuild/interactive.yaml index 51cb8c82a2f..4aa56e0fadc 100644 --- a/src/tests/ftest/rebuild/interactive.yaml +++ b/src/tests/ftest/rebuild/interactive.yaml @@ -43,7 +43,7 @@ ior: test_file: /testFile api: DFS transfer_size: 1M - block_size: 256M + block_size: 128M flags_write: "-v -w -k -G 1 -F" flags_read: "-v -r -R -k -G 1 -F" diff --git a/src/tests/ftest/util/ior_utils.py b/src/tests/ftest/util/ior_utils.py index ff11a375259..bd2984f31cf 100644 --- a/src/tests/ftest/util/ior_utils.py +++ b/src/tests/ftest/util/ior_utils.py @@ -133,7 +133,7 @@ def thread_run_ior(thread_queue, job_id, test, manager, log, hosts, path, slots, pool, container, processes, ppn, intercept, plugin_path, dfuse, display_space, fail_on_warning, namespace, ior_params) - except CommandFailure as error: + except Exception as error: # pylint: disable=broad-except thread_result["result"] = CmdResult(command="", stdout=str(error), exit_status=1) finally: manager.verbose = saved_verbose