From 296eea8577a094a5f33e7ff1d63da74c02df7ba6 Mon Sep 17 00:00:00 2001 From: Henry Ruhs Date: Fri, 31 Jan 2025 10:17:33 +0100 Subject: [PATCH] Feat/halt on error (#862) * Introduce halt-on-error * Introduce halt-on-error * Fix wording --- facefusion.ini | 1 + facefusion/args.py | 1 + facefusion/core.py | 8 ++--- facefusion/jobs/job_manager.py | 18 +++++++---- facefusion/jobs/job_runner.py | 18 +++++++---- facefusion/program.py | 38 ++++++++++++++---------- facefusion/typing.py | 2 ++ facefusion/uis/components/job_manager.py | 10 +++++++ facefusion/uis/components/job_runner.py | 10 +++++-- facefusion/wording.py | 1 + tests/test_cli_job_manager.py | 8 ++--- tests/test_cli_job_runner.py | 14 ++++----- tests/test_job_manager.py | 15 ++++++---- tests/test_job_runner.py | 9 +++--- 14 files changed, 99 insertions(+), 54 deletions(-) diff --git a/facefusion.ini b/facefusion.ini index 96b006e..b81c657 100644 --- a/facefusion.ini +++ b/facefusion.ini @@ -115,3 +115,4 @@ system_memory_limit = [misc] log_level = +halt_on_error = diff --git a/facefusion/args.py b/facefusion/args.py index 5538a4b..7cccdaf 100644 --- a/facefusion/args.py +++ b/facefusion/args.py @@ -129,6 +129,7 @@ def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None: apply_state_item('system_memory_limit', args.get('system_memory_limit')) # misc apply_state_item('log_level', args.get('log_level')) + apply_state_item('halt_on_error', args.get('halt_on_error')) # jobs apply_state_item('job_id', args.get('job_id')) apply_state_item('job_status', args.get('job_status')) diff --git a/facefusion/core.py b/facefusion/core.py index 19ac0cd..f2e980e 100755 --- a/facefusion/core.py +++ b/facefusion/core.py @@ -183,7 +183,7 @@ def route_job_manager(args : Args) -> ErrorCode: return 1 if state_manager.get_item('command') == 'job-submit-all': - if job_manager.submit_jobs(): + if job_manager.submit_jobs(state_manager.get_item('halt_on_error')): logger.info(wording.get('job_all_submitted'), __name__) return 0 logger.error(wording.get('job_all_not_submitted'), __name__) @@ -197,7 +197,7 @@ def route_job_manager(args : Args) -> ErrorCode: return 1 if state_manager.get_item('command') == 'job-delete-all': - if job_manager.delete_jobs(): + if job_manager.delete_jobs(state_manager.get_item('halt_on_error')): logger.info(wording.get('job_all_deleted'), __name__) return 0 logger.error(wording.get('job_all_not_deleted'), __name__) @@ -250,7 +250,7 @@ def route_job_runner() -> ErrorCode: if state_manager.get_item('command') == 'job-run-all': logger.info(wording.get('running_jobs'), __name__) - if job_runner.run_jobs(process_step): + if job_runner.run_jobs(process_step, state_manager.get_item('halt_on_error')): logger.info(wording.get('processing_jobs_succeed'), __name__) return 0 logger.info(wording.get('processing_jobs_failed'), __name__) @@ -266,7 +266,7 @@ def route_job_runner() -> ErrorCode: if state_manager.get_item('command') == 'job-retry-all': logger.info(wording.get('retrying_jobs'), __name__) - if job_runner.retry_jobs(process_step): + if job_runner.retry_jobs(process_step, state_manager.get_item('halt_on_error')): logger.info(wording.get('processing_jobs_succeed'), __name__) return 0 logger.info(wording.get('processing_jobs_failed'), __name__) diff --git a/facefusion/jobs/job_manager.py b/facefusion/jobs/job_manager.py index dedad90..a503c86 100644 --- a/facefusion/jobs/job_manager.py +++ b/facefusion/jobs/job_manager.py @@ -48,14 +48,17 @@ def submit_job(job_id : str) -> bool: return False -def submit_jobs() -> bool: +def submit_jobs(halt_on_error : bool) -> bool: drafted_job_ids = find_job_ids('drafted') + has_error = False if drafted_job_ids: for job_id in drafted_job_ids: if not submit_job(job_id): - return False - return True + has_error = True + if halt_on_error: + return False + return not has_error return False @@ -63,14 +66,17 @@ def delete_job(job_id : str) -> bool: return delete_job_file(job_id) -def delete_jobs() -> bool: +def delete_jobs(halt_on_error : bool) -> bool: job_ids = find_job_ids('drafted') + find_job_ids('queued') + find_job_ids('failed') + find_job_ids('completed') + has_error = False if job_ids: for job_id in job_ids: if not delete_job(job_id): - return False - return True + has_error = True + if halt_on_error: + return False + return not has_error return False diff --git a/facefusion/jobs/job_runner.py b/facefusion/jobs/job_runner.py index 33a2faf..ab6f92a 100644 --- a/facefusion/jobs/job_runner.py +++ b/facefusion/jobs/job_runner.py @@ -16,14 +16,17 @@ def run_job(job_id : str, process_step : ProcessStep) -> bool: return False -def run_jobs(process_step : ProcessStep) -> bool: +def run_jobs(process_step : ProcessStep, halt_on_error : bool) -> bool: queued_job_ids = job_manager.find_job_ids('queued') + has_error = False if queued_job_ids: for job_id in queued_job_ids: if not run_job(job_id, process_step): - return False - return True + has_error = True + if halt_on_error: + return False + return not has_error return False @@ -35,14 +38,17 @@ def retry_job(job_id : str, process_step : ProcessStep) -> bool: return False -def retry_jobs(process_step : ProcessStep) -> bool: +def retry_jobs(process_step : ProcessStep, halt_on_error : bool) -> bool: failed_job_ids = job_manager.find_job_ids('failed') + has_error = False if failed_job_ids: for job_id in failed_job_ids: if not retry_job(job_id, process_step): - return False - return True + has_error = True + if halt_on_error: + return False + return not has_error return False diff --git a/facefusion/program.py b/facefusion/program.py index 41e220f..0667204 100755 --- a/facefusion/program.py +++ b/facefusion/program.py @@ -230,7 +230,7 @@ def create_memory_program() -> ArgumentParser: return program -def create_misc_program() -> ArgumentParser: +def create_log_level_program() -> ArgumentParser: program = ArgumentParser(add_help = False) group_misc = program.add_argument_group('misc') group_misc.add_argument('--log-level', help = wording.get('help.log_level'), default = config.get_str_value('misc.log_level', 'info'), choices = facefusion.choices.log_levels) @@ -238,6 +238,14 @@ def create_misc_program() -> ArgumentParser: return program +def create_halt_on_error_program() -> ArgumentParser: + program = ArgumentParser(add_help = False) + group_misc = program.add_argument_group('misc') + group_misc.add_argument('--halt-on-error', help = wording.get('help.halt_on_error'), action = 'store_true', default = config.get_bool_value('misc.halt_on_error')) + job_store.register_job_keys([ 'halt_on_error' ]) + return program + + def create_job_id_program() -> ArgumentParser: program = ArgumentParser(add_help = False) program.add_argument('job_id', help = wording.get('help.job_id')) @@ -262,7 +270,7 @@ def collect_step_program() -> ArgumentParser: def collect_job_program() -> ArgumentParser: - return ArgumentParser(parents= [ create_execution_program(), create_download_providers_program(), create_memory_program(), create_misc_program() ], add_help = False) + return ArgumentParser(parents= [ create_execution_program(), create_download_providers_program(), create_memory_program(), create_log_level_program() ], add_help = False) def create_program() -> ArgumentParser: @@ -274,23 +282,23 @@ def create_program() -> ArgumentParser: sub_program.add_parser('run', help = wording.get('help.run'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_uis_program(), collect_job_program() ], formatter_class = create_help_formatter_large) sub_program.add_parser('headless-run', help = wording.get('help.headless_run'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), collect_job_program() ], formatter_class = create_help_formatter_large) sub_program.add_parser('batch-run', help = wording.get('help.batch_run'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), create_source_pattern_program(), create_target_pattern_program(), create_output_pattern_program(), collect_step_program(), collect_job_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('force-download', help = wording.get('help.force_download'), parents = [ create_download_providers_program(), create_download_scope_program(), create_misc_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('force-download', help = wording.get('help.force_download'), parents = [ create_download_providers_program(), create_download_scope_program(), create_log_level_program() ], formatter_class = create_help_formatter_large) # job manager - sub_program.add_parser('job-list', help = wording.get('help.job_list'), parents = [ create_job_status_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('job-create', help = wording.get('help.job_create'), parents = [ create_job_id_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('job-submit', help = wording.get('help.job_submit'), parents = [ create_job_id_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('job-submit-all', help = wording.get('help.job_submit_all'), parents = [ create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('job-delete', help = wording.get('help.job_delete'), parents = [ create_job_id_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('job-delete-all', help = wording.get('help.job_delete_all'), parents = [ create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('job-add-step', help = wording.get('help.job_add_step'), parents = [ create_job_id_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_misc_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('job-remix-step', help = wording.get('help.job_remix_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_output_path_program(), collect_step_program(), create_misc_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('job-insert-step', help = wording.get('help.job_insert_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_misc_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('job-remove-step', help = wording.get('help.job_remove_step'), parents = [ create_job_id_program(), create_step_index_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-list', help = wording.get('help.job_list'), parents = [ create_job_status_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-create', help = wording.get('help.job_create'), parents = [ create_job_id_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-submit', help = wording.get('help.job_submit'), parents = [ create_job_id_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-submit-all', help = wording.get('help.job_submit_all'), parents = [ create_jobs_path_program(), create_log_level_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-delete', help = wording.get('help.job_delete'), parents = [ create_job_id_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-delete-all', help = wording.get('help.job_delete_all'), parents = [ create_jobs_path_program(), create_log_level_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-add-step', help = wording.get('help.job_add_step'), parents = [ create_job_id_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_log_level_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-remix-step', help = wording.get('help.job_remix_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_output_path_program(), collect_step_program(), create_log_level_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-insert-step', help = wording.get('help.job_insert_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_log_level_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-remove-step', help = wording.get('help.job_remove_step'), parents = [ create_job_id_program(), create_step_index_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large) # job runner sub_program.add_parser('job-run', help = wording.get('help.job_run'), parents = [ create_job_id_program(), create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('job-run-all', help = wording.get('help.job_run_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-run-all', help = wording.get('help.job_run_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large) sub_program.add_parser('job-retry', help = wording.get('help.job_retry'), parents = [ create_job_id_program(), create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large) - sub_program.add_parser('job-retry-all', help = wording.get('help.job_retry_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large) + sub_program.add_parser('job-retry-all', help = wording.get('help.job_retry_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large) return ArgumentParser(parents = [ program ], formatter_class = create_help_formatter_small) diff --git a/facefusion/typing.py b/facefusion/typing.py index 10ef6ec..03e5483 100755 --- a/facefusion/typing.py +++ b/facefusion/typing.py @@ -283,6 +283,7 @@ StateKey = Literal\ 'video_memory_strategy', 'system_memory_limit', 'log_level', + 'halt_on_error', 'job_id', 'job_status', 'step_index' @@ -347,6 +348,7 @@ State = TypedDict('State', 'video_memory_strategy' : VideoMemoryStrategy, 'system_memory_limit' : int, 'log_level' : LogLevel, + 'halt_on_error' : bool, 'job_id' : str, 'job_status' : JobStatus, 'step_index' : int diff --git a/facefusion/uis/components/job_manager.py b/facefusion/uis/components/job_manager.py index 893e433..727aaf9 100644 --- a/facefusion/uis/components/job_manager.py +++ b/facefusion/uis/components/job_manager.py @@ -89,6 +89,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id : if is_directory(step_args.get('output_path')): step_args['output_path'] = suggest_output_path(step_args.get('output_path'), state_manager.get_item('target_path')) + if job_action == 'job-create': if created_job_id and job_manager.create_job(created_job_id): updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ] @@ -97,6 +98,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id : return gradio.Dropdown(value = 'job-add-step'), gradio.Textbox(visible = False), gradio.Dropdown(value = created_job_id, choices = updated_job_ids, visible = True), gradio.Dropdown() else: logger.error(wording.get('job_not_created').format(job_id = created_job_id), __name__) + if job_action == 'job-submit': if selected_job_id and job_manager.submit_job(selected_job_id): updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ] @@ -105,6 +107,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id : return gradio.Dropdown(), gradio.Textbox(), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids, visible = True), gradio.Dropdown() else: logger.error(wording.get('job_not_submitted').format(job_id = selected_job_id), __name__) + if job_action == 'job-delete': if selected_job_id and job_manager.delete_job(selected_job_id): updated_job_ids = job_manager.find_job_ids('drafted') + job_manager.find_job_ids('queued') + job_manager.find_job_ids('failed') + job_manager.find_job_ids('completed') or [ 'none' ] @@ -113,6 +116,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id : return gradio.Dropdown(), gradio.Textbox(), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids, visible = True), gradio.Dropdown() else: logger.error(wording.get('job_not_deleted').format(job_id = selected_job_id), __name__) + if job_action == 'job-add-step': if selected_job_id and job_manager.add_step(selected_job_id, step_args): state_manager.set_item('output_path', output_path) @@ -121,6 +125,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id : else: state_manager.set_item('output_path', output_path) logger.error(wording.get('job_step_not_added').format(job_id = selected_job_id), __name__) + if job_action == 'job-remix-step': if selected_job_id and job_manager.has_step(selected_job_id, selected_step_index) and job_manager.remix_step(selected_job_id, selected_step_index, step_args): updated_step_choices = get_step_choices(selected_job_id) or [ 'none' ] #type:ignore[list-item] @@ -131,6 +136,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id : else: state_manager.set_item('output_path', output_path) logger.error(wording.get('job_remix_step_not_added').format(job_id = selected_job_id, step_index = selected_step_index), __name__) + if job_action == 'job-insert-step': if selected_job_id and job_manager.has_step(selected_job_id, selected_step_index) and job_manager.insert_step(selected_job_id, selected_step_index, step_args): updated_step_choices = get_step_choices(selected_job_id) or [ 'none' ] #type:ignore[list-item] @@ -141,6 +147,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id : else: state_manager.set_item('output_path', output_path) logger.error(wording.get('job_step_not_inserted').format(job_id = selected_job_id, step_index = selected_step_index), __name__) + if job_action == 'job-remove-step': if selected_job_id and job_manager.has_step(selected_job_id, selected_step_index) and job_manager.remove_step(selected_job_id, selected_step_index): updated_step_choices = get_step_choices(selected_job_id) or [ 'none' ] #type:ignore[list-item] @@ -160,16 +167,19 @@ def get_step_choices(job_id : str) -> List[int]: def update(job_action : JobManagerAction, selected_job_id : str) -> Tuple[gradio.Textbox, gradio.Dropdown, gradio.Dropdown]: if job_action == 'job-create': return gradio.Textbox(value = None, visible = True), gradio.Dropdown(visible = False), gradio.Dropdown(visible = False) + if job_action == 'job-delete': updated_job_ids = job_manager.find_job_ids('drafted') + job_manager.find_job_ids('queued') + job_manager.find_job_ids('failed') + job_manager.find_job_ids('completed') or [ 'none' ] updated_job_id = selected_job_id if selected_job_id in updated_job_ids else get_last(updated_job_ids) return gradio.Textbox(visible = False), gradio.Dropdown(value = updated_job_id, choices = updated_job_ids, visible = True), gradio.Dropdown(visible = False) + if job_action in [ 'job-submit', 'job-add-step' ]: updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ] updated_job_id = selected_job_id if selected_job_id in updated_job_ids else get_last(updated_job_ids) return gradio.Textbox(visible = False), gradio.Dropdown(value = updated_job_id, choices = updated_job_ids, visible = True), gradio.Dropdown(visible = False) + if job_action in [ 'job-remix-step', 'job-insert-step', 'job-remove-step' ]: updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ] updated_job_id = selected_job_id if selected_job_id in updated_job_ids else get_last(updated_job_ids) diff --git a/facefusion/uis/components/job_runner.py b/facefusion/uis/components/job_runner.py index 540a102..90bccb2 100644 --- a/facefusion/uis/components/job_runner.py +++ b/facefusion/uis/components/job_runner.py @@ -95,12 +95,15 @@ def run(job_action : JobRunnerAction, job_id : str) -> Tuple[gradio.Button, grad updated_job_ids = job_manager.find_job_ids('queued') or [ 'none' ] return gradio.Button(visible = True), gradio.Button(visible = False), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids) + if job_action == 'job-run-all': logger.info(wording.get('running_jobs'), __name__) - if job_runner.run_jobs(process_step): + halt_on_error = False + if job_runner.run_jobs(process_step, halt_on_error): logger.info(wording.get('processing_jobs_succeed'), __name__) else: logger.info(wording.get('processing_jobs_failed'), __name__) + if job_action == 'job-retry': logger.info(wording.get('retrying_job').format(job_id = job_id), __name__) if job_id and job_runner.retry_job(job_id, process_step): @@ -110,9 +113,11 @@ def run(job_action : JobRunnerAction, job_id : str) -> Tuple[gradio.Button, grad updated_job_ids = job_manager.find_job_ids('failed') or [ 'none' ] return gradio.Button(visible = True), gradio.Button(visible = False), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids) + if job_action == 'job-retry-all': logger.info(wording.get('retrying_jobs'), __name__) - if job_runner.retry_jobs(process_step): + halt_on_error = False + if job_runner.retry_jobs(process_step, halt_on_error): logger.info(wording.get('processing_jobs_succeed'), __name__) else: logger.info(wording.get('processing_jobs_failed'), __name__) @@ -129,6 +134,7 @@ def update_job_action(job_action : JobRunnerAction) -> gradio.Dropdown: updated_job_ids = job_manager.find_job_ids('queued') or [ 'none' ] return gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids, visible = True) + if job_action == 'job-retry': updated_job_ids = job_manager.find_job_ids('failed') or [ 'none' ] diff --git a/facefusion/wording.py b/facefusion/wording.py index 93a8572..8cc3c58 100755 --- a/facefusion/wording.py +++ b/facefusion/wording.py @@ -200,6 +200,7 @@ WORDING : Dict[str, Any] =\ 'system_memory_limit': 'limit the available RAM that can be used while processing', # misc 'log_level': 'adjust the message severity displayed in the terminal', + 'halt_on_error': 'halt the program once an error occurred', # run 'run': 'run the program', 'headless_run': 'run the program in headless mode', diff --git a/tests/test_cli_job_manager.py b/tests/test_cli_job_manager.py index fac789c..a20a272 100644 --- a/tests/test_cli_job_manager.py +++ b/tests/test_cli_job_manager.py @@ -61,7 +61,7 @@ def test_job_submit() -> None: def test_submit_all() -> None: - commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory() ] + commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ] assert subprocess.run(commands).returncode == 1 @@ -79,7 +79,7 @@ def test_submit_all() -> None: commands = [ sys.executable, 'facefusion.py', 'job-add-step', 'test-job-submit-all-2', '--jobs-path', get_test_jobs_directory(), '-s', get_test_example_file('source.jpg'), '-t', get_test_example_file('target-240p.jpg'), '-o', get_test_output_file('test-job-remix-step.jpg') ] subprocess.run(commands) - commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory() ] + commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ] assert subprocess.run(commands).returncode == 0 assert is_test_job_file('test-job-submit-all-1.json', 'queued') is True @@ -103,7 +103,7 @@ def test_job_delete() -> None: def test_job_delete_all() -> None: - commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory() ] + commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ] assert subprocess.run(commands).returncode == 1 @@ -113,7 +113,7 @@ def test_job_delete_all() -> None: commands = [ sys.executable, 'facefusion.py', 'job-create', 'test-job-delete-all-2', '--jobs-path', get_test_jobs_directory() ] subprocess.run(commands) - commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory() ] + commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ] assert subprocess.run(commands).returncode == 0 assert is_test_job_file('test-job-delete-all-1.json', 'drafted') is False diff --git a/tests/test_cli_job_runner.py b/tests/test_cli_job_runner.py index 89dbad5..23d23cb 100644 --- a/tests/test_cli_job_runner.py +++ b/tests/test_cli_job_runner.py @@ -51,7 +51,7 @@ def test_job_run() -> None: def test_job_run_all() -> None: - commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory() ] + commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ] assert subprocess.run(commands).returncode == 1 @@ -70,14 +70,14 @@ def test_job_run_all() -> None: commands = [ sys.executable, 'facefusion.py', 'job-add-step', 'test-job-run-all-2', '--jobs-path', get_test_jobs_directory(), '--processors', 'face_debugger', '-t', get_test_example_file('target-240p.mp4'), '-o', get_test_output_file('test-job-run-all-2.mp4'), '--trim-frame-start', '0', '--trim-frame-end', '1' ] subprocess.run(commands) - commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory() ] + commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ] assert subprocess.run(commands).returncode == 1 - commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory() ] + commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ] subprocess.run(commands) - commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory() ] + commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ] assert subprocess.run(commands).returncode == 0 assert subprocess.run(commands).returncode == 1 @@ -111,7 +111,7 @@ def test_job_retry() -> None: def test_job_retry_all() -> None: - commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory() ] + commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ] assert subprocess.run(commands).returncode == 1 @@ -130,7 +130,7 @@ def test_job_retry_all() -> None: commands = [ sys.executable, 'facefusion.py', 'job-add-step', 'test-job-retry-all-2', '--jobs-path', get_test_jobs_directory(), '--processors', 'face_debugger', '-t', get_test_example_file('target-240p.mp4'), '-o', get_test_output_file('test-job-retry-all-2.mp4'), '--trim-frame-start', '0', '--trim-frame-end', '1' ] subprocess.run(commands) - commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory() ] + commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ] assert subprocess.run(commands).returncode == 1 @@ -139,7 +139,7 @@ def test_job_retry_all() -> None: move_job_file('test-job-retry-all-1', 'failed') move_job_file('test-job-retry-all-2', 'failed') - commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory() ] + commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ] assert subprocess.run(commands).returncode == 0 assert subprocess.run(commands).returncode == 1 diff --git a/tests/test_job_manager.py b/tests/test_job_manager.py index 3ee6c0d..3de99ec 100644 --- a/tests/test_job_manager.py +++ b/tests/test_job_manager.py @@ -63,19 +63,20 @@ def test_submit_jobs() -> None: 'target_path': 'target-2.jpg', 'output_path': 'output-2.jpg' } + halt_on_error = True - assert submit_jobs() is False + assert submit_jobs(halt_on_error) is False create_job('job-test-submit-jobs-1') create_job('job-test-submit-jobs-2') - assert submit_jobs() is False + assert submit_jobs(halt_on_error) is False add_step('job-test-submit-jobs-1', args_1) add_step('job-test-submit-jobs-2', args_2) - assert submit_jobs() is True - assert submit_jobs() is False + assert submit_jobs(halt_on_error) is True + assert submit_jobs(halt_on_error) is False def test_delete_job() -> None: @@ -88,12 +89,14 @@ def test_delete_job() -> None: def test_delete_jobs() -> None: - assert delete_jobs() is False + halt_on_error = True + + assert delete_jobs(halt_on_error) is False create_job('job-test-delete-jobs-1') create_job('job-test-delete-jobs-2') - assert delete_jobs() is True + assert delete_jobs(halt_on_error) is True @pytest.mark.skip() diff --git a/tests/test_job_runner.py b/tests/test_job_runner.py index a51968d..7c14207 100644 --- a/tests/test_job_runner.py +++ b/tests/test_job_runner.py @@ -85,8 +85,9 @@ def test_run_jobs() -> None: 'target_path': get_test_example_file('target-240p.jpg'), 'output_path': get_test_output_file('output-1.jpg') } + halt_on_error = True - assert run_jobs(process_step) is False + assert run_jobs(process_step, halt_on_error) is False create_job('job-test-run-jobs-1') create_job('job-test-run-jobs-2') @@ -95,11 +96,11 @@ def test_run_jobs() -> None: add_step('job-test-run-jobs-2', args_2) add_step('job-test-run-jobs-3', args_3) - assert run_jobs(process_step) is False + assert run_jobs(process_step, halt_on_error) is False - submit_jobs() + submit_jobs(halt_on_error) - assert run_jobs(process_step) is True + assert run_jobs(process_step, halt_on_error) is True @pytest.mark.skip()