Merge pull request #900 from facefusion/vibe/benchmark-command

Benchmark command
This commit is contained in:
Henry Ruhs
2025-06-16 13:12:49 +02:00
committed by GitHub
13 changed files with 192 additions and 114 deletions

View File

@@ -37,6 +37,7 @@ commands:
headless-run run the program in headless mode
batch-run run the program in batch mode
force-download force automate downloads and exit
benchmark benchmark the program
job-list list jobs by status
job-create create a drafted job
job-submit submit a drafted job to become a queued job

View File

@@ -110,6 +110,10 @@ execution_queue_count =
download_providers =
download_scope =
[benchmark]
benchmark_resolutions =
benchmark_cycles =
[memory]
video_memory_strategy =
system_memory_limit =

View File

@@ -125,6 +125,9 @@ def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:
# download
apply_state_item('download_providers', args.get('download_providers'))
apply_state_item('download_scope', args.get('download_scope'))
# benchmark
apply_state_item('benchmark_resolutions', args.get('benchmark_resolutions'))
apply_state_item('benchmark_cycles', args.get('benchmark_cycles'))
# memory
apply_state_item('video_memory_strategy', args.get('video_memory_strategy'))
apply_state_item('system_memory_limit', args.get('system_memory_limit'))

106
facefusion/benchmarker.py Normal file
View File

@@ -0,0 +1,106 @@
import hashlib
import os
import statistics
import tempfile
from time import perf_counter
from typing import Generator, List
import facefusion.choices
from facefusion import core, state_manager
from facefusion.cli_helper import render_table
from facefusion.download import conditional_download, resolve_download_url
from facefusion.filesystem import get_file_extension
from facefusion.types import BenchmarkCycleSet
from facefusion.vision import count_video_frame_total, detect_video_fps, detect_video_resolution, pack_resolution
def pre_check() -> bool:
conditional_download('.assets/examples',
[
resolve_download_url('examples-3.0.0', 'source.jpg'),
resolve_download_url('examples-3.0.0', 'source.mp3'),
resolve_download_url('examples-3.0.0', 'target-240p.mp4'),
resolve_download_url('examples-3.0.0', 'target-360p.mp4'),
resolve_download_url('examples-3.0.0', 'target-540p.mp4'),
resolve_download_url('examples-3.0.0', 'target-720p.mp4'),
resolve_download_url('examples-3.0.0', 'target-1080p.mp4'),
resolve_download_url('examples-3.0.0', 'target-1440p.mp4'),
resolve_download_url('examples-3.0.0', 'target-2160p.mp4')
])
return True
def run() -> Generator[List[BenchmarkCycleSet], None, None]:
benchmark_resolutions = state_manager.get_item('benchmark_resolutions')
benchmark_cycles = state_manager.get_item('benchmark_cycles')
state_manager.set_item('source_paths', [ '.assets/examples/source.jpg', '.assets/examples/source.mp3' ])
state_manager.set_item('face_landmarker_score', 0)
state_manager.set_item('temp_frame_format', 'bmp')
state_manager.set_item('output_audio_volume', 0)
state_manager.set_item('output_video_preset', 'ultrafast')
state_manager.set_item('video_memory_strategy', 'tolerant')
benchmarks = []
target_paths = [facefusion.choices.benchmark_set.get(benchmark_resolution) for benchmark_resolution in benchmark_resolutions if benchmark_resolution in facefusion.choices.benchmark_set]
for target_path in target_paths:
state_manager.set_item('target_path', target_path)
state_manager.set_item('output_path', suggest_output_path(state_manager.get_item('target_path')))
benchmarks.append(cycle(benchmark_cycles))
yield benchmarks
def cycle(benchmark_cycles : int) -> BenchmarkCycleSet:
process_times = []
video_frame_total = count_video_frame_total(state_manager.get_item('target_path'))
output_video_resolution = detect_video_resolution(state_manager.get_item('target_path'))
state_manager.set_item('output_video_resolution', pack_resolution(output_video_resolution))
state_manager.set_item('output_video_fps', detect_video_fps(state_manager.get_item('target_path')))
core.conditional_process()
for index in range(benchmark_cycles):
start_time = perf_counter()
core.conditional_process()
end_time = perf_counter()
process_times.append(end_time - start_time)
average_run = round(statistics.mean(process_times), 2)
fastest_run = round(min(process_times), 2)
slowest_run = round(max(process_times), 2)
relative_fps = round(video_frame_total * benchmark_cycles / sum(process_times), 2)
return\
{
'target_path': state_manager.get_item('target_path'),
'benchmark_cycles': benchmark_cycles,
'average_run': average_run,
'fastest_run': fastest_run,
'slowest_run': slowest_run,
'relative_fps': relative_fps
}
def suggest_output_path(target_path : str) -> str:
target_file_extension = get_file_extension(target_path)
return os.path.join(tempfile.gettempdir(), hashlib.sha1().hexdigest()[:8] + target_file_extension)
def render() -> None:
benchmarks = []
headers =\
[
'target_path',
'benchmark_cycles',
'average_run',
'fastest_run',
'slowest_run',
'relative_fps'
]
for benchmark in run():
benchmarks = benchmark
contents = [ list(benchmark_set.values()) for benchmark_set in benchmarks ]
render_table(headers, contents)

View File

@@ -2,7 +2,7 @@ import logging
from typing import List, Sequence
from facefusion.common_helper import create_float_range, create_int_range
from facefusion.types import Angle, AudioEncoder, AudioFormat, AudioTypeSet, DownloadProvider, DownloadProviderSet, DownloadScope, EncoderSet, ExecutionProvider, ExecutionProviderSet, FaceDetectorModel, FaceDetectorSet, FaceLandmarkerModel, FaceMaskArea, FaceMaskAreaSet, FaceMaskRegion, FaceMaskRegionSet, FaceMaskType, FaceOccluderModel, FaceParserModel, FaceSelectorMode, FaceSelectorOrder, Gender, ImageFormat, ImageTypeSet, JobStatus, LogLevel, LogLevelSet, Race, Score, TempFrameFormat, UiWorkflow, VideoEncoder, VideoFormat, VideoMemoryStrategy, VideoPreset, VideoTypeSet, WebcamMode
from facefusion.types import Angle, AudioEncoder, AudioFormat, AudioTypeSet, BenchmarkResolution, BenchmarkSet, DownloadProvider, DownloadProviderSet, DownloadScope, EncoderSet, ExecutionProvider, ExecutionProviderSet, FaceDetectorModel, FaceDetectorSet, FaceLandmarkerModel, FaceMaskArea, FaceMaskAreaSet, FaceMaskRegion, FaceMaskRegionSet, FaceMaskType, FaceOccluderModel, FaceParserModel, FaceSelectorMode, FaceSelectorOrder, Gender, ImageFormat, ImageTypeSet, JobStatus, LogLevel, LogLevelSet, Race, Score, TempFrameFormat, UiWorkflow, VideoEncoder, VideoFormat, VideoMemoryStrategy, VideoPreset, VideoTypeSet, WebcamMode
face_detector_set : FaceDetectorSet =\
{
@@ -85,6 +85,18 @@ output_video_presets : List[VideoPreset] = [ 'ultrafast', 'superfast', 'veryfast
image_template_sizes : List[float] = [ 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4 ]
video_template_sizes : List[int] = [ 240, 360, 480, 540, 720, 1080, 1440, 2160, 4320 ]
benchmark_set : BenchmarkSet =\
{
'240p': '.assets/examples/target-240p.mp4',
'360p': '.assets/examples/target-360p.mp4',
'540p': '.assets/examples/target-540p.mp4',
'720p': '.assets/examples/target-720p.mp4',
'1080p': '.assets/examples/target-1080p.mp4',
'1440p': '.assets/examples/target-1440p.mp4',
'2160p': '.assets/examples/target-2160p.mp4'
}
benchmark_resolutions : List[BenchmarkResolution] = list(benchmark_set.keys())
webcam_modes : List[WebcamMode] = [ 'inline', 'udp', 'v4l2' ]
webcam_resolutions : List[str] = [ '320x240', '640x480', '800x600', '1024x768', '1280x720', '1280x960', '1920x1080', '2560x1440', '3840x2160' ]
@@ -136,6 +148,7 @@ log_levels : List[LogLevel] = list(log_level_set.keys())
ui_workflows : List[UiWorkflow] = [ 'instant_runner', 'job_runner', 'job_manager' ]
job_statuses : List[JobStatus] = [ 'drafted', 'queued', 'completed', 'failed' ]
benchmark_cycles_range : Sequence[int] = create_int_range(1, 10, 1)
execution_thread_count_range : Sequence[int] = create_int_range(1, 32, 1)
execution_queue_count_range : Sequence[int] = create_int_range(1, 4, 1)
system_memory_limit_range : Sequence[int] = create_int_range(0, 128, 4)

View File

@@ -6,7 +6,7 @@ from time import time
import numpy
from facefusion import cli_helper, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, logger, process_manager, state_manager, video_manager, voice_extractor, wording
from facefusion import benchmarker, cli_helper, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, logger, process_manager, state_manager, video_manager, voice_extractor, wording
from facefusion.args import apply_args, collect_job_args, reduce_job_args, reduce_step_args
from facefusion.common_helper import get_first
from facefusion.content_analyser import analyse_image, analyse_video
@@ -59,6 +59,11 @@ def route(args : Args) -> None:
error_code = force_download()
return hard_exit(error_code)
if state_manager.get_item('command') == 'benchmark':
if not common_pre_check() or not processors_pre_check() or not benchmarker.pre_check():
return hard_exit(2)
benchmarker.render()
if state_manager.get_item('command') in [ 'job-list', 'job-create', 'job-submit', 'job-submit-all', 'job-delete', 'job-delete-all', 'job-add-step', 'job-remix-step', 'job-insert-step', 'job-remove-step' ]:
if not job_manager.init_jobs(state_manager.get_item('jobs_path')):
hard_exit(1)

View File

@@ -214,6 +214,14 @@ def create_download_providers_program() -> ArgumentParser:
return program
def create_benchmark_program() -> ArgumentParser:
program = ArgumentParser(add_help = False)
group_benchmark = program.add_argument_group('benchmark')
group_benchmark.add_argument('--benchmark-resolutions', help = wording.get('help.benchmark_resolutions'), default = config.get_str_list('benchmark', 'benchmark_resolutions', get_first(facefusion.choices.benchmark_resolutions)), choices = facefusion.choices.benchmark_resolutions, nargs = '+')
group_benchmark.add_argument('--benchmark-cycles', help = wording.get('help.benchmark_cycles'), type = int, default = config.get_int_value('benchmark', 'benchmark_cycles', '5'), choices = facefusion.choices.benchmark_cycles_range)
return program
def create_download_scope_program() -> ArgumentParser:
program = ArgumentParser(add_help = False)
group_download = program.add_argument_group('download')
@@ -284,6 +292,7 @@ def create_program() -> ArgumentParser:
sub_program.add_parser('headless-run', help = wording.get('help.headless_run'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), collect_job_program() ], formatter_class = create_help_formatter_large)
sub_program.add_parser('batch-run', help = wording.get('help.batch_run'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), create_source_pattern_program(), create_target_pattern_program(), create_output_pattern_program(), collect_step_program(), collect_job_program() ], formatter_class = create_help_formatter_large)
sub_program.add_parser('force-download', help = wording.get('help.force_download'), parents = [ create_download_providers_program(), create_download_scope_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)
sub_program.add_parser('benchmark', help = wording.get('help.benchmark'), parents = [ create_temp_path_program(), create_benchmark_program(), collect_step_program(), collect_job_program() ], formatter_class = create_help_formatter_large)
# job manager
sub_program.add_parser('job-list', help = wording.get('help.job_list'), parents = [ create_job_status_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)
sub_program.add_parser('job-create', help = wording.get('help.job_create'), parents = [ create_job_id_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)

View File

@@ -130,6 +130,18 @@ EncoderSet = TypedDict('EncoderSet',
})
VideoPreset = Literal['ultrafast', 'superfast', 'veryfast', 'faster', 'fast', 'medium', 'slow', 'slower', 'veryslow']
BenchmarkResolution = Literal['240p', '360p', '540p', '720p', '1080p', '1440p', '2160p']
BenchmarkSet : TypeAlias = Dict[BenchmarkResolution, str]
BenchmarkCycleSet = TypedDict('BenchmarkCycleSet',
{
'target_path' : str,
'benchmark_cycles' : int,
'average_run' : float,
'fastest_run' : float,
'slowest_run' : float,
'relative_fps' : float
})
WebcamMode = Literal['inline', 'udp', 'v4l2']
StreamMode = Literal['udp', 'v4l2']
@@ -238,6 +250,10 @@ StateKey = Literal\
'source_pattern',
'target_pattern',
'output_pattern',
'download_providers',
'download_scope',
'benchmark_resolutions',
'benchmark_cycles',
'face_detector_model',
'face_detector_size',
'face_detector_angles',
@@ -282,8 +298,6 @@ StateKey = Literal\
'execution_providers',
'execution_thread_count',
'execution_queue_count',
'download_providers',
'download_scope',
'video_memory_strategy',
'system_memory_limit',
'log_level',
@@ -304,6 +318,10 @@ State = TypedDict('State',
'source_pattern' : str,
'target_pattern' : str,
'output_pattern' : str,
'download_providers': List[DownloadProvider],
'download_scope': DownloadScope,
'benchmark_resolutions': List[BenchmarkResolution],
'benchmark_cycles': int,
'face_detector_model' : FaceDetectorModel,
'face_detector_size' : str,
'face_detector_angles' : List[Angle],
@@ -348,8 +366,6 @@ State = TypedDict('State',
'execution_providers' : List[ExecutionProvider],
'execution_thread_count' : int,
'execution_queue_count' : int,
'download_providers' : List[DownloadProvider],
'download_scope' : DownloadScope,
'video_memory_strategy' : VideoMemoryStrategy,
'system_memory_limit' : int,
'log_level' : LogLevel,

View File

@@ -1,31 +1,13 @@
import hashlib
import os
import statistics
import tempfile
from time import perf_counter
from typing import Any, Dict, Generator, List, Optional
from typing import Any, Generator, List, Optional
import gradio
from facefusion import state_manager, wording
from facefusion.core import conditional_process
from facefusion.filesystem import get_file_extension, is_video
from facefusion.memory import limit_system_memory
from facefusion import benchmarker, state_manager, wording
from facefusion.types import BenchmarkResolution
from facefusion.uis.core import get_ui_component
from facefusion.vision import count_video_frame_total, detect_video_fps, detect_video_resolution, pack_resolution
BENCHMARK_BENCHMARKS_DATAFRAME : Optional[gradio.Dataframe] = None
BENCHMARK_START_BUTTON : Optional[gradio.Button] = None
BENCHMARKS : Dict[str, str] =\
{
'240p': '.assets/examples/target-240p.mp4',
'360p': '.assets/examples/target-360p.mp4',
'540p': '.assets/examples/target-540p.mp4',
'720p': '.assets/examples/target-720p.mp4',
'1080p': '.assets/examples/target-1080p.mp4',
'1440p': '.assets/examples/target-1440p.mp4',
'2160p': '.assets/examples/target-2160p.mp4'
}
def render() -> None:
@@ -61,72 +43,19 @@ def render() -> None:
def listen() -> None:
benchmark_runs_checkbox_group = get_ui_component('benchmark_runs_checkbox_group')
benchmark_resolutions_checkbox_group = get_ui_component('benchmark_resolutions_checkbox_group')
benchmark_cycles_slider = get_ui_component('benchmark_cycles_slider')
if benchmark_runs_checkbox_group and benchmark_cycles_slider:
BENCHMARK_START_BUTTON.click(start, inputs = [ benchmark_runs_checkbox_group, benchmark_cycles_slider ], outputs = BENCHMARK_BENCHMARKS_DATAFRAME)
if benchmark_resolutions_checkbox_group and benchmark_cycles_slider:
BENCHMARK_START_BUTTON.click(start, inputs = [ benchmark_resolutions_checkbox_group, benchmark_cycles_slider ], outputs = BENCHMARK_BENCHMARKS_DATAFRAME)
def suggest_output_path(target_path : str) -> Optional[str]:
if is_video(target_path):
target_file_extension = get_file_extension(target_path)
return os.path.join(tempfile.gettempdir(), hashlib.sha1().hexdigest()[:8] + target_file_extension)
return None
def start(benchmark_runs : List[str], benchmark_cycles : int) -> Generator[List[Any], None, None]:
state_manager.init_item('source_paths', [ '.assets/examples/source.jpg', '.assets/examples/source.mp3' ])
state_manager.init_item('face_landmarker_score', 0)
state_manager.init_item('temp_frame_format', 'bmp')
state_manager.init_item('output_audio_volume', 0)
state_manager.init_item('output_video_preset', 'ultrafast')
def start(benchmark_resolutions : List[BenchmarkResolution], benchmark_cycles : int) -> Generator[List[Any], None, None]:
state_manager.set_item('benchmark_resolutions', benchmark_resolutions)
state_manager.set_item('benchmark_cycles', benchmark_cycles)
state_manager.sync_item('execution_providers')
state_manager.sync_item('execution_thread_count')
state_manager.sync_item('execution_queue_count')
state_manager.sync_item('system_memory_limit')
benchmark_results = []
target_paths = [ BENCHMARKS[benchmark_run] for benchmark_run in benchmark_runs if benchmark_run in BENCHMARKS ]
if target_paths:
pre_process()
for target_path in target_paths:
state_manager.init_item('target_path', target_path)
state_manager.init_item('output_path', suggest_output_path(state_manager.get_item('target_path')))
benchmark_results.append(benchmark(benchmark_cycles))
yield benchmark_results
def pre_process() -> None:
system_memory_limit = state_manager.get_item('system_memory_limit')
if system_memory_limit and system_memory_limit > 0:
limit_system_memory(system_memory_limit)
def benchmark(benchmark_cycles : int) -> List[Any]:
process_times = []
video_frame_total = count_video_frame_total(state_manager.get_item('target_path'))
output_video_resolution = detect_video_resolution(state_manager.get_item('target_path'))
state_manager.init_item('output_video_resolution', pack_resolution(output_video_resolution))
state_manager.init_item('output_video_fps', detect_video_fps(state_manager.get_item('target_path')))
conditional_process()
for index in range(benchmark_cycles):
start_time = perf_counter()
conditional_process()
end_time = perf_counter()
process_times.append(end_time - start_time)
average_run = round(statistics.mean(process_times), 2)
fastest_run = round(min(process_times), 2)
slowest_run = round(max(process_times), 2)
relative_fps = round(video_frame_total * benchmark_cycles / sum(process_times), 2)
return\
[
state_manager.get_item('target_path'),
benchmark_cycles,
average_run,
fastest_run,
slowest_run,
relative_fps
]
for benchmark in benchmarker.run():
yield [ list(benchmark_set.values()) for benchmark_set in benchmark ]

View File

@@ -2,29 +2,29 @@ from typing import Optional
import gradio
import facefusion.choices
from facefusion import wording
from facefusion.uis.components.benchmark import BENCHMARKS
from facefusion.uis.core import register_ui_component
BENCHMARK_RUNS_CHECKBOX_GROUP : Optional[gradio.CheckboxGroup] = None
BENCHMARK_RESOLUTIONS_CHECKBOX_GROUP : Optional[gradio.CheckboxGroup] = None
BENCHMARK_CYCLES_SLIDER : Optional[gradio.Button] = None
def render() -> None:
global BENCHMARK_RUNS_CHECKBOX_GROUP
global BENCHMARK_RESOLUTIONS_CHECKBOX_GROUP
global BENCHMARK_CYCLES_SLIDER
BENCHMARK_RUNS_CHECKBOX_GROUP = gradio.CheckboxGroup(
label = wording.get('uis.benchmark_runs_checkbox_group'),
choices = list(BENCHMARKS.keys()),
value = list(BENCHMARKS.keys())
BENCHMARK_RESOLUTIONS_CHECKBOX_GROUP = gradio.CheckboxGroup(
label = wording.get('uis.benchmark_resolutions_checkbox_group'),
choices = facefusion.choices.benchmark_resolutions,
value = facefusion.choices.benchmark_resolutions
)
BENCHMARK_CYCLES_SLIDER = gradio.Slider(
label = wording.get('uis.benchmark_cycles_slider'),
value = 5,
step = 1,
minimum = 1,
maximum = 10
minimum = min(facefusion.choices.benchmark_cycles_range),
maximum = max(facefusion.choices.benchmark_cycles_range)
)
register_ui_component('benchmark_runs_checkbox_group', BENCHMARK_RUNS_CHECKBOX_GROUP)
register_ui_component('benchmark_resolutions_checkbox_group', BENCHMARK_RESOLUTIONS_CHECKBOX_GROUP)
register_ui_component('benchmark_cycles_slider', BENCHMARK_CYCLES_SLIDER)

View File

@@ -1,24 +1,12 @@
import gradio
from facefusion import state_manager
from facefusion.download import conditional_download, resolve_download_url
from facefusion.benchmarker import pre_check as benchmarker_pre_check
from facefusion.uis.components import about, age_modifier_options, benchmark, benchmark_options, deep_swapper_options, download, execution, execution_queue_count, execution_thread_count, expression_restorer_options, face_debugger_options, face_editor_options, face_enhancer_options, face_swapper_options, frame_colorizer_options, frame_enhancer_options, lip_syncer_options, memory, processors
def pre_check() -> bool:
conditional_download('.assets/examples',
[
resolve_download_url('examples-3.0.0', 'source.jpg'),
resolve_download_url('examples-3.0.0', 'source.mp3'),
resolve_download_url('examples-3.0.0', 'target-240p.mp4'),
resolve_download_url('examples-3.0.0', 'target-360p.mp4'),
resolve_download_url('examples-3.0.0', 'target-540p.mp4'),
resolve_download_url('examples-3.0.0', 'target-720p.mp4'),
resolve_download_url('examples-3.0.0', 'target-1080p.mp4'),
resolve_download_url('examples-3.0.0', 'target-1440p.mp4'),
resolve_download_url('examples-3.0.0', 'target-2160p.mp4')
])
return True
return benchmarker_pre_check()
def render() -> gradio.Blocks:

View File

@@ -6,7 +6,7 @@ ComponentName = Literal\
'age_modifier_direction_slider',
'age_modifier_model_dropdown',
'benchmark_cycles_slider',
'benchmark_runs_checkbox_group',
'benchmark_resolutions_checkbox_group',
'deep_swapper_model_dropdown',
'deep_swapper_morph_slider',
'expression_restorer_factor_slider',

View File

@@ -197,6 +197,9 @@ WORDING : Dict[str, Any] =\
# download
'download_providers': 'download using different providers (choices: {choices}, ...)',
'download_scope': 'specify the download scope',
# benchmark
'benchmark_resolutions': 'choose the resolution for the benchmarks (choices: {choices}, ...)',
'benchmark_cycles': 'specify the number of benchmark cycles',
# memory
'video_memory_strategy': 'balance fast processing and low VRAM usage',
'system_memory_limit': 'limit the available RAM that can be used while processing',
@@ -208,6 +211,7 @@ WORDING : Dict[str, Any] =\
'headless_run': 'run the program in headless mode',
'batch_run': 'run the program in batch mode',
'force_download': 'force automate downloads and exit',
'benchmark': 'benchmark the program',
# jobs
'job_id': 'specify the job id',
'job_status': 'specify the job status',
@@ -241,7 +245,7 @@ WORDING : Dict[str, Any] =\
'age_modifier_model_dropdown': 'AGE MODIFIER MODEL',
'apply_button': 'APPLY',
'benchmark_cycles_slider': 'BENCHMARK CYCLES',
'benchmark_runs_checkbox_group': 'BENCHMARK RUNS',
'benchmark_resolutions_checkbox_group': 'BENCHMARK RESOLUTIONS',
'clear_button': 'CLEAR',
'common_options_checkbox_group': 'OPTIONS',
'download_providers_checkbox_group': 'DOWNLOAD PROVIDERS',