From 62483fdfad5be91f072345b6d19371e05f7f662c Mon Sep 17 00:00:00 2001 From: henryruhs Date: Thu, 22 May 2025 09:26:36 +0200 Subject: [PATCH 1/4] Introduce video manager to handle broken videos --- facefusion/face_store.py | 4 +- facefusion/processors/modules/age_modifier.py | 3 +- facefusion/processors/modules/deep_swapper.py | 3 +- .../processors/modules/expression_restorer.py | 3 +- .../processors/modules/face_debugger.py | 3 +- facefusion/processors/modules/face_editor.py | 3 +- .../processors/modules/face_enhancer.py | 3 +- facefusion/processors/modules/face_swapper.py | 5 +- .../processors/modules/frame_colorizer.py | 3 +- .../processors/modules/frame_enhancer.py | 3 +- facefusion/processors/modules/lip_syncer.py | 3 +- facefusion/types.py | 2 + facefusion/video_manager.py | 21 +++++++++ facefusion/vision.py | 46 ++++++++++++------- 14 files changed, 75 insertions(+), 30 deletions(-) create mode 100644 facefusion/video_manager.py diff --git a/facefusion/face_store.py b/facefusion/face_store.py index 518b9e3..59e6d5d 100644 --- a/facefusion/face_store.py +++ b/facefusion/face_store.py @@ -30,7 +30,7 @@ def set_static_faces(vision_frame : VisionFrame, faces : List[Face]) -> None: def clear_static_faces() -> None: - FACE_STORE['static_faces'] = {} + FACE_STORE['static_faces'].clear() def create_frame_hash(vision_frame : VisionFrame) -> Optional[str]: @@ -53,4 +53,4 @@ def append_reference_face(name : str, face : Face) -> None: def clear_reference_faces() -> None: - FACE_STORE['reference_faces'] = {} + FACE_STORE['reference_faces'].clear() diff --git a/facefusion/processors/modules/age_modifier.py b/facefusion/processors/modules/age_modifier.py index 925808b..4719744 100755 --- a/facefusion/processors/modules/age_modifier.py +++ b/facefusion/processors/modules/age_modifier.py @@ -9,7 +9,7 @@ import facefusion.choices import facefusion.jobs.job_manager import facefusion.jobs.job_store import facefusion.processors.core as processors -from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording +from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wording from facefusion.common_helper import create_int_metavar from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url from facefusion.execution import has_execution_provider @@ -115,6 +115,7 @@ def pre_process(mode : ProcessMode) -> bool: def post_process() -> None: read_static_image.cache_clear() + video_manager.clear_video_pool() if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]: clear_inference_pool() if state_manager.get_item('video_memory_strategy') == 'strict': diff --git a/facefusion/processors/modules/deep_swapper.py b/facefusion/processors/modules/deep_swapper.py index 14e2370..54438c7 100755 --- a/facefusion/processors/modules/deep_swapper.py +++ b/facefusion/processors/modules/deep_swapper.py @@ -9,7 +9,7 @@ from cv2.typing import Size import facefusion.jobs.job_manager import facefusion.jobs.job_store import facefusion.processors.core as processors -from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording +from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wording from facefusion.common_helper import create_int_metavar from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url_by_provider from facefusion.face_analyser import get_many_faces, get_one_face @@ -311,6 +311,7 @@ def pre_process(mode : ProcessMode) -> bool: def post_process() -> None: read_static_image.cache_clear() + video_manager.clear_video_pool() if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]: clear_inference_pool() if state_manager.get_item('video_memory_strategy') == 'strict': diff --git a/facefusion/processors/modules/expression_restorer.py b/facefusion/processors/modules/expression_restorer.py index 4d301a4..0e307dd 100755 --- a/facefusion/processors/modules/expression_restorer.py +++ b/facefusion/processors/modules/expression_restorer.py @@ -8,7 +8,7 @@ import numpy import facefusion.jobs.job_manager import facefusion.jobs.job_store import facefusion.processors.core as processors -from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording +from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wording from facefusion.common_helper import create_int_metavar from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url from facefusion.face_analyser import get_many_faces, get_one_face @@ -129,6 +129,7 @@ def pre_process(mode : ProcessMode) -> bool: def post_process() -> None: read_static_image.cache_clear() + video_manager.clear_video_pool() if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]: clear_inference_pool() if state_manager.get_item('video_memory_strategy') == 'strict': diff --git a/facefusion/processors/modules/face_debugger.py b/facefusion/processors/modules/face_debugger.py index e4397f2..ea53b7d 100755 --- a/facefusion/processors/modules/face_debugger.py +++ b/facefusion/processors/modules/face_debugger.py @@ -7,7 +7,7 @@ import numpy import facefusion.jobs.job_manager import facefusion.jobs.job_store import facefusion.processors.core as processors -from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, logger, process_manager, state_manager, wording +from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, logger, process_manager, state_manager, video_manager, wording from facefusion.face_analyser import get_many_faces, get_one_face from facefusion.face_helper import warp_face_by_face_landmark_5 from facefusion.face_masker import create_occlusion_mask, create_region_mask, create_static_box_mask @@ -56,6 +56,7 @@ def pre_process(mode : ProcessMode) -> bool: def post_process() -> None: read_static_image.cache_clear() + video_manager.clear_video_pool() if state_manager.get_item('video_memory_strategy') == 'strict': content_analyser.clear_inference_pool() face_classifier.clear_inference_pool() diff --git a/facefusion/processors/modules/face_editor.py b/facefusion/processors/modules/face_editor.py index 3f494d8..2414d14 100755 --- a/facefusion/processors/modules/face_editor.py +++ b/facefusion/processors/modules/face_editor.py @@ -8,7 +8,7 @@ import numpy import facefusion.jobs.job_manager import facefusion.jobs.job_store import facefusion.processors.core as processors -from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording +from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wording from facefusion.common_helper import create_float_metavar from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url from facefusion.face_analyser import get_many_faces, get_one_face @@ -182,6 +182,7 @@ def pre_process(mode : ProcessMode) -> bool: def post_process() -> None: read_static_image.cache_clear() + video_manager.clear_video_pool() if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]: clear_inference_pool() if state_manager.get_item('video_memory_strategy') == 'strict': diff --git a/facefusion/processors/modules/face_enhancer.py b/facefusion/processors/modules/face_enhancer.py index 1787d69..ad1b0a5 100755 --- a/facefusion/processors/modules/face_enhancer.py +++ b/facefusion/processors/modules/face_enhancer.py @@ -8,7 +8,7 @@ import numpy import facefusion.jobs.job_manager import facefusion.jobs.job_store import facefusion.processors.core as processors -from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording +from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wording from facefusion.common_helper import create_float_metavar, create_int_metavar from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url from facefusion.face_analyser import get_many_faces, get_one_face @@ -275,6 +275,7 @@ def pre_process(mode : ProcessMode) -> bool: def post_process() -> None: read_static_image.cache_clear() + video_manager.clear_video_pool() if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]: clear_inference_pool() if state_manager.get_item('video_memory_strategy') == 'strict': diff --git a/facefusion/processors/modules/face_swapper.py b/facefusion/processors/modules/face_swapper.py index 7ce0ed4..1df1fa0 100755 --- a/facefusion/processors/modules/face_swapper.py +++ b/facefusion/processors/modules/face_swapper.py @@ -8,7 +8,7 @@ import facefusion.choices import facefusion.jobs.job_manager import facefusion.jobs.job_store import facefusion.processors.core as processors -from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording +from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wording from facefusion.common_helper import get_first from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url from facefusion.execution import has_execution_provider @@ -406,9 +406,10 @@ def pre_process(mode : ProcessMode) -> bool: def post_process() -> None: read_static_image.cache_clear() + video_manager.clear_video_pool() if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]: - clear_inference_pool() get_static_model_initializer.cache_clear() + clear_inference_pool() if state_manager.get_item('video_memory_strategy') == 'strict': content_analyser.clear_inference_pool() face_classifier.clear_inference_pool() diff --git a/facefusion/processors/modules/frame_colorizer.py b/facefusion/processors/modules/frame_colorizer.py index 12a1942..abf3fc9 100644 --- a/facefusion/processors/modules/frame_colorizer.py +++ b/facefusion/processors/modules/frame_colorizer.py @@ -8,7 +8,7 @@ import numpy import facefusion.jobs.job_manager import facefusion.jobs.job_store import facefusion.processors.core as processors -from facefusion import config, content_analyser, inference_manager, logger, process_manager, state_manager, wording +from facefusion import config, content_analyser, inference_manager, logger, process_manager, state_manager, video_manager, wording from facefusion.common_helper import create_int_metavar from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url from facefusion.execution import has_execution_provider @@ -188,6 +188,7 @@ def pre_process(mode : ProcessMode) -> bool: def post_process() -> None: read_static_image.cache_clear() + video_manager.clear_video_pool() if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]: clear_inference_pool() if state_manager.get_item('video_memory_strategy') == 'strict': diff --git a/facefusion/processors/modules/frame_enhancer.py b/facefusion/processors/modules/frame_enhancer.py index 5b653a7..27a9b6c 100644 --- a/facefusion/processors/modules/frame_enhancer.py +++ b/facefusion/processors/modules/frame_enhancer.py @@ -8,7 +8,7 @@ import numpy import facefusion.jobs.job_manager import facefusion.jobs.job_store import facefusion.processors.core as processors -from facefusion import config, content_analyser, inference_manager, logger, process_manager, state_manager, wording +from facefusion import config, content_analyser, inference_manager, logger, process_manager, state_manager, video_manager, wording from facefusion.common_helper import create_int_metavar from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url from facefusion.execution import has_execution_provider @@ -450,6 +450,7 @@ def pre_process(mode : ProcessMode) -> bool: def post_process() -> None: read_static_image.cache_clear() + video_manager.clear_video_pool() if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]: clear_inference_pool() if state_manager.get_item('video_memory_strategy') == 'strict': diff --git a/facefusion/processors/modules/lip_syncer.py b/facefusion/processors/modules/lip_syncer.py index 1b6faa2..ac38cbb 100755 --- a/facefusion/processors/modules/lip_syncer.py +++ b/facefusion/processors/modules/lip_syncer.py @@ -8,7 +8,7 @@ import numpy import facefusion.jobs.job_manager import facefusion.jobs.job_store import facefusion.processors.core as processors -from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, voice_extractor, wording +from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, voice_extractor, wording from facefusion.audio import create_empty_audio_frame, get_voice_frame, read_static_voice from facefusion.common_helper import get_first from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url @@ -127,6 +127,7 @@ def pre_process(mode : ProcessMode) -> bool: def post_process() -> None: read_static_image.cache_clear() read_static_voice.cache_clear() + video_manager.clear_video_pool() if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]: clear_inference_pool() if state_manager.get_item('video_memory_strategy') == 'strict': diff --git a/facefusion/types.py b/facefusion/types.py index def3aa0..57210b0 100755 --- a/facefusion/types.py +++ b/facefusion/types.py @@ -1,6 +1,7 @@ from collections import namedtuple from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, TypeAlias, TypedDict +import cv2 import numpy from numpy.typing import NDArray from onnxruntime import InferenceSession @@ -49,6 +50,7 @@ FaceStore = TypedDict('FaceStore', 'static_faces' : FaceSet, 'reference_faces' : FaceSet }) +VideoPoolSet : TypeAlias = Dict[str, cv2.VideoCapture] VisionFrame : TypeAlias = NDArray[Any] Mask : TypeAlias = NDArray[Any] diff --git a/facefusion/video_manager.py b/facefusion/video_manager.py new file mode 100644 index 0000000..f38bc19 --- /dev/null +++ b/facefusion/video_manager.py @@ -0,0 +1,21 @@ +import cv2 + +from facefusion.types import VideoPoolSet + +VIDEO_POOL_SET : VideoPoolSet = {} + + +def get_video_capture(video_path : str) -> cv2.VideoCapture: + if video_path not in VIDEO_POOL_SET: + VIDEO_POOL_SET[video_path] = cv2.VideoCapture(video_path) + + return VIDEO_POOL_SET.get(video_path) + + +def clear_video_pool() -> None: + global VIDEO_POOL_SET + + for video_capture in VIDEO_POOL_SET.values(): + video_capture.release() + + VIDEO_POOL_SET.clear() diff --git a/facefusion/vision.py b/facefusion/vision.py index d03e383..b146170 100644 --- a/facefusion/vision.py +++ b/facefusion/vision.py @@ -9,7 +9,9 @@ from cv2.typing import Size import facefusion.choices from facefusion.common_helper import is_windows from facefusion.filesystem import get_file_extension, is_image, is_video +from facefusion.thread_helper import thread_semaphore from facefusion.types import Duration, Fps, Orientation, Resolution, VisionFrame +from facefusion.video_manager import get_video_capture @lru_cache() @@ -81,24 +83,30 @@ def create_image_resolutions(resolution : Resolution) -> List[str]: def read_video_frame(video_path : str, frame_number : int = 0) -> Optional[VisionFrame]: if is_video(video_path): - video_capture = cv2.VideoCapture(video_path) + video_capture = get_video_capture(video_path) + if video_capture.isOpened(): frame_total = video_capture.get(cv2.CAP_PROP_FRAME_COUNT) - video_capture.set(cv2.CAP_PROP_POS_FRAMES, min(frame_total, frame_number - 1)) - has_vision_frame, vision_frame = video_capture.read() - video_capture.release() + + with thread_semaphore(): + video_capture.set(cv2.CAP_PROP_POS_FRAMES, min(frame_total, frame_number - 1)) + has_vision_frame, vision_frame = video_capture.read() + if has_vision_frame: return vision_frame + return None def count_video_frame_total(video_path : str) -> int: if is_video(video_path): - video_capture = cv2.VideoCapture(video_path) + video_capture = get_video_capture(video_path) + if video_capture.isOpened(): - video_frame_total = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT)) - video_capture.release() - return video_frame_total + with thread_semaphore(): + video_frame_total = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT)) + return video_frame_total + return 0 @@ -112,11 +120,13 @@ def predict_video_frame_total(video_path : str, fps : Fps, trim_frame_start : in def detect_video_fps(video_path : str) -> Optional[float]: if is_video(video_path): - video_capture = cv2.VideoCapture(video_path) + video_capture = get_video_capture(video_path) + if video_capture.isOpened(): - video_fps = video_capture.get(cv2.CAP_PROP_FPS) - video_capture.release() - return video_fps + with thread_semaphore(): + video_fps = video_capture.get(cv2.CAP_PROP_FPS) + return video_fps + return None @@ -163,12 +173,14 @@ def restrict_trim_frame(video_path : str, trim_frame_start : Optional[int], trim def detect_video_resolution(video_path : str) -> Optional[Resolution]: if is_video(video_path): - video_capture = cv2.VideoCapture(video_path) + video_capture = get_video_capture(video_path) + if video_capture.isOpened(): - width = video_capture.get(cv2.CAP_PROP_FRAME_WIDTH) - height = video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT) - video_capture.release() - return int(width), int(height) + with thread_semaphore(): + width = video_capture.get(cv2.CAP_PROP_FRAME_WIDTH) + height = video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT) + return int(width), int(height) + return None From f50cb7409985bf79cfc1dc7e312db8d303a14745 Mon Sep 17 00:00:00 2001 From: henryruhs Date: Thu, 22 May 2025 09:29:15 +0200 Subject: [PATCH 2/4] Introduce video manager to handle broken videos --- facefusion/video_manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/facefusion/video_manager.py b/facefusion/video_manager.py index f38bc19..d687caf 100644 --- a/facefusion/video_manager.py +++ b/facefusion/video_manager.py @@ -13,8 +13,6 @@ def get_video_capture(video_path : str) -> cv2.VideoCapture: def clear_video_pool() -> None: - global VIDEO_POOL_SET - for video_capture in VIDEO_POOL_SET.values(): video_capture.release() From 4343bf562992565c3ccf241925ba2ba7b0a8d9ab Mon Sep 17 00:00:00 2001 From: henryruhs Date: Thu, 22 May 2025 08:49:12 +0200 Subject: [PATCH 3/4] Fix moving temp files for Windows --- facefusion/core.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/facefusion/core.py b/facefusion/core.py index 03a2b9a..90c5558 100755 --- a/facefusion/core.py +++ b/facefusion/core.py @@ -6,7 +6,7 @@ from time import time import numpy -from facefusion import cli_helper, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, logger, process_manager, state_manager, voice_extractor, wording +from facefusion import cli_helper, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, logger, process_manager, state_manager, voice_extractor, wording, video_manager from facefusion.args import apply_args, collect_job_args, reduce_job_args, reduce_step_args from facefusion.common_helper import get_first from facefusion.content_analyser import analyse_image, analyse_video @@ -468,8 +468,10 @@ def process_video(start_time : float) -> ErrorCode: source_audio_path = get_first(filter_audio_paths(state_manager.get_item('source_paths'))) if source_audio_path: if replace_audio(state_manager.get_item('target_path'), source_audio_path, state_manager.get_item('output_path')): + video_manager.clear_video_pool() logger.debug(wording.get('replacing_audio_succeed'), __name__) else: + video_manager.clear_video_pool() if is_process_stopping(): process_manager.end() return 4 @@ -477,8 +479,10 @@ def process_video(start_time : float) -> ErrorCode: move_temp_file(state_manager.get_item('target_path'), state_manager.get_item('output_path')) else: if restore_audio(state_manager.get_item('target_path'), state_manager.get_item('output_path'), trim_frame_start, trim_frame_end): + video_manager.clear_video_pool() logger.debug(wording.get('restoring_audio_succeed'), __name__) else: + video_manager.clear_video_pool() if is_process_stopping(): process_manager.end() return 4 From 42e72f8d7eb8d643be0fcd01c5f84967bc31c7fd Mon Sep 17 00:00:00 2001 From: henryruhs Date: Thu, 22 May 2025 11:00:50 +0200 Subject: [PATCH 4/4] Fix CI --- facefusion/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/facefusion/core.py b/facefusion/core.py index 90c5558..837d97e 100755 --- a/facefusion/core.py +++ b/facefusion/core.py @@ -6,7 +6,7 @@ from time import time import numpy -from facefusion import cli_helper, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, logger, process_manager, state_manager, voice_extractor, wording, video_manager +from facefusion import cli_helper, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, logger, process_manager, state_manager, video_manager, voice_extractor, wording from facefusion.args import apply_args, collect_job_args, reduce_job_args, reduce_step_args from facefusion.common_helper import get_first from facefusion.content_analyser import analyse_image, analyse_video