Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
cbac21a
Lock processor settings during DLC inference
C-Achard Jun 30, 2026
fd6df70
Improve processor discovery and logging
C-Achard Jul 1, 2026
b58a953
Add processors package exports
C-Achard Jul 1, 2026
f3402f9
Move example socket processors to examples module
C-Achard Jul 1, 2026
867f758
Skip socket base module in processor scan
C-Achard Jul 1, 2026
a4e839a
Update processor_utils.py
C-Achard Jul 1, 2026
02ef6a5
Warn on duplicate processor registration
C-Achard Jul 1, 2026
e1dcead
Fix dlclive Processor import paths
C-Achard Jul 1, 2026
664823c
Extract processor registry into new module
C-Achard Jul 1, 2026
6c63b07
Refactor camera worker and recording pipeline
C-Achard Jul 2, 2026
015d068
Route recording frames through recording sink
C-Achard Jul 2, 2026
9c66e7f
Update recording manager test imports
C-Achard Jul 2, 2026
1a8b62b
Propagate capture metadata in single-camera flow
C-Achard Jul 2, 2026
a72bdb6
Add timestamp metadata to frame signal
C-Achard Jul 3, 2026
0d015c6
Comment previous signals
C-Achard Jul 3, 2026
dea3890
Fix dispatcher lifecycle and add flush API
C-Achard Jul 3, 2026
7947e2c
Update tests for CapturedFrame integration
C-Achard Jul 3, 2026
5ee3394
Stabilize recording and camera tests
C-Achard Jul 3, 2026
04abe92
Harden recording dispatcher shutdown flow
C-Achard Aug 14, 2026
e040f20
Retry recorder stop until success
C-Achard Aug 14, 2026
68e6e17
Disable debug timing log
C-Achard Aug 14, 2026
97a8e1c
Add timeout for recorder stop retries
C-Achard Aug 18, 2026
5caf032
Rename recording frame toggle API
C-Achard Aug 18, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion dlclivegui/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,16 @@
DEFAULT_RECORDING_FPS: float = 30.0
ALLOWED_VIDEO_CONTAINERS: set[str] = {"mp4", "avi", "mov"}
DEFAULT_RECORDING_CONTAINER: str = "mp4"
RECORD_STOP_RETRY_INTERVAL: float = 0.25
RECORD_STOP_RETRY_TIMEOUT: float = 5.0


## Debug
### Timing logs
SINGLE_CAMERA_WORKER_DO_LOG_TIMING: bool = False
MULTI_CAMERA_WORKER_DO_LOG_TIMING: bool = False
REC_DO_LOG_TIMING: bool = False
DLC_DO_LOG_TIMING: bool = True
DLC_DO_LOG_TIMING: bool = False
### Trigger debug logging
DEBUG_TRIGGER_LOGS = False
# MAIN_WINDOW_DO_LOG_TIMING: bool = False
Expand Down
5 changes: 3 additions & 2 deletions dlclivegui/gui/camera_config/preview.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

from PySide6.QtCore import QTimer

from ...services.camera_controller import SingleCameraWorker
from ...services.multi_camera_controller import MultiCameraController

if TYPE_CHECKING:
Expand Down Expand Up @@ -56,7 +57,7 @@ class PreviewSession:


def apply_rotation(frame, rotation):
return MultiCameraController.apply_rotation(frame, rotation)
return SingleCameraWorker.apply_rotation(frame, rotation)


def apply_crop(frame, x0, y0, x1, y1):
Expand All @@ -66,7 +67,7 @@ def apply_crop(frame, x0, y0, x1, y1):
x1 = max(x0, min(x1, w))
y1 = max(y0, min(y1, h))

return MultiCameraController.apply_crop(frame, (x0, y0, x1, y1))
return SingleCameraWorker.apply_crop(frame, (x0, y0, x1, y1))


def resize_to_fit(frame, max_w=400, max_h=300):
Expand Down
36 changes: 28 additions & 8 deletions dlclivegui/gui/main_window.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@
DEFAULT_RECORDING_CONTAINER,
DLC_DO_LOG_TIMING,
GUI_MAX_DISPLAY_FPS,
RECORD_STOP_RETRY_INTERVAL,
RECORD_STOP_RETRY_TIMEOUT,
ApplicationSettings,
BoundingBoxSettings,
CameraSettings,
Expand All @@ -71,6 +73,7 @@
)
from ..services.dlc_processor import DLCLiveProcessor, PoseResult
from ..services.multi_camera_controller import MultiCameraController, MultiFrameData, get_camera_id, get_display_id
from ..services.recording_manager import RecordingManager
from ..utils.display import BBoxColors, compute_tile_info, create_tiled_frame, draw_bbox, draw_pose
from ..utils.settings_store import DLCLiveGUISettingsStore, ModelPathStore
from ..utils.stats import WorkerTimingStats, format_dlc_stats
Expand All @@ -80,7 +83,6 @@
from .misc import layouts as lyts
from .misc.drag_spinbox import ScrubSpinBox
from .misc.eliding_label import ElidingPathLabel
from .recording_manager import RecordingManager
from .theme import LOGO, LOGO_ALPHA, AppStyle, apply_theme

logger = logging.getLogger("DLCLiveGUI")
Expand Down Expand Up @@ -812,7 +814,7 @@ def _connect_signals(self) -> None:
# Multi-camera controller signals (used for both single and multi-camera modes)
self.multi_camera_controller.frame_ready.connect(self._on_multi_frame_processing_ready)
self.multi_camera_controller.display_ready.connect(self._on_multi_frame_display_ready)
self.multi_camera_controller.recording_frame_ready.connect(self._on_recording_frame_ready)
# self.multi_camera_controller.recording_frame_ready.connect(self._on_recording_frame_ready)
self.multi_camera_controller.all_started.connect(self._on_multi_camera_started)
self.multi_camera_controller.all_stopped.connect(self._on_multi_camera_stopped)
self.multi_camera_controller.camera_error.connect(self._on_multi_camera_error)
Expand Down Expand Up @@ -1621,7 +1623,8 @@ def _start_multi_camera_recording(self) -> None:
if run_dir is None:
self._show_error("Failed to start recording.")
return
self.multi_camera_controller.set_recording_frame_do_emit(True)
self.multi_camera_controller.set_recording_sink(self._rec_manager.write_frame)
self.multi_camera_controller.set_recording_frame_is_enabled(True)

self._settings_store.set_session_name(session_name)
self.start_record_button.setEnabled(False)
Expand All @@ -1644,15 +1647,26 @@ def _stop_multi_camera_recording(self) -> None:

# Stop frame emission immediately so no new frames enter recording pipeline.
try:
self.multi_camera_controller.set_recording_frame_do_emit(False)
self.multi_camera_controller.set_recording_frame_is_enabled(False)
self.multi_camera_controller.set_recording_sink(None)
except Exception:
logger.exception("Failed to disable recording frame emission")

def worker():
total_wait_time = 0.0
try:
self._rec_manager.stop_all()
finally:
self._recording_stopped_async.emit()
while not self._rec_manager.stop_all():
logger.info("Retrying recorder stop...")
time.sleep(RECORD_STOP_RETRY_INTERVAL)
total_wait_time += RECORD_STOP_RETRY_INTERVAL
if total_wait_time >= RECORD_STOP_RETRY_TIMEOUT:
logger.error("Timeout while stopping recording after %.1f seconds", total_wait_time)
raise RuntimeError("Could not stop recording within timeout period.")
except Exception as e:
Comment thread
C-Achard marked this conversation as resolved.
logger.exception("Error while stopping recording: %s", e)
return

self._recording_stopped_async.emit()

threading.Thread(
target=worker,
Expand Down Expand Up @@ -2259,7 +2273,13 @@ def closeEvent(self, event: QCloseEvent) -> None: # pragma: no cover - GUI beha
if hasattr(self, "_camera_validation_timer") and self._camera_validation_timer.isActive():
self._camera_validation_timer.stop()
# Stop all multi-camera recorders
self._rec_manager.stop_all()
try:
self.multi_camera_controller.set_recording_frame_is_enabled(False)
except Exception:
logger.exception("Failed to disable recording frame emission during shutdown")
while not self._rec_manager.stop_all():
logger.info("Retrying recorder stop during shutdown...")
time.sleep(RECORD_STOP_RETRY_INTERVAL)

# Close the camera dialog if open (ensures its worker thread is canceled)
if getattr(self, "_cam_dialog", None) is not None and self._cam_dialog.isVisible():
Expand Down
258 changes: 258 additions & 0 deletions dlclivegui/services/camera_controller.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,258 @@
from __future__ import annotations

import copy
import logging
import time
from threading import Event, Lock

import cv2
import numpy as np
from PySide6.QtCore import QObject, Signal, Slot

from dlclivegui.cameras import CameraFactory
from dlclivegui.cameras.base import CameraBackend

# from dlclivegui.config import CameraSettings
from dlclivegui.config import (
SINGLE_CAMERA_WORKER_DO_LOG_TIMING,
CameraSettings,
)
from dlclivegui.utils.stats import WorkerTimingStats

logger = logging.getLogger(__name__)


class SingleCameraWorker(QObject):
"""Worker for a single camera in multi-camera mode."""

frame_captured = Signal(str, object, float, object) # camera_id, frame, timestamp, timestamp_metadata
error_occurred = Signal(str, str) # camera_id, error_message
runtime_info = Signal(str, object) # camera_id, dict of runtime info
started = Signal(str) # camera_id
stopped = Signal(str) # camera_id

def __init__(self, camera_id: str, settings: CameraSettings):
super().__init__()
self._camera_id = camera_id
self._settings = copy.deepcopy(settings)
self._stop_event = Event()
self._backend: CameraBackend | None = None
self._max_consecutive_errors = 5
self._retry_delay = 0.1
self._trigger_timeout_delay = 0.05
self._trigger_wait_log_interval = 2.0
self._last_trigger_wait_log = 0.0
self._trigger_wait_suppressed_count = 0

self._recording_sink = None
self._recording_enabled = False
self._recording_sink_lock = Lock()

# Performance logs
self._timing = WorkerTimingStats(
camera_id, logger=logger, log_interval=1.0, enabled=SINGLE_CAMERA_WORKER_DO_LOG_TIMING
)

def set_recording_sink(self, sink) -> None:
with self._recording_sink_lock:
self._recording_sink = sink

def set_recording_enabled(self, enabled: bool) -> None:
with self._recording_sink_lock:
self._recording_enabled = bool(enabled)

@Slot()
def run(self) -> None:
self._stop_event.clear()

try:
logger.debug(
"[Worker %s] before create: backend=%s index=%s properties=%s",
self._camera_id,
self._settings.backend,
self._settings.index,
self._settings.properties,
)

self._backend = CameraFactory.create(self._settings)

logger.debug(
"[Worker %s] after create: backend=%s index=%s properties=%s",
self._camera_id,
self._backend.settings.backend,
self._backend.settings.index,
self._backend.settings.properties,
)

self._backend.open()
self.runtime_info.emit(
self._camera_id,
{
"actual_fps": getattr(self._backend, "actual_fps", None),
"actual_resolution": getattr(self._backend, "actual_resolution", None),
"actual_pixel_format": getattr(self._backend, "actual_pixel_format", None),
"actual_output_format": getattr(self._backend, "actual_output_format", None),
},
)
except Exception as exc:
logger.exception(f"Failed to initialize camera {self._camera_id}", exc_info=exc)
self.error_occurred.emit(self._camera_id, f"Failed to initialize camera: {exc}")
self.stopped.emit(self._camera_id)
return

self.started.emit(self._camera_id)
consecutive_errors = 0

while not self._stop_event.is_set():
try:
with self._timing.measure("Single.read"):
captured = self._backend.read()
frame = captured.frame
timestamp = captured.software_timestamp
timestamp_metadata = captured.timestamp_metadata
if frame is None or frame.size == 0:
consecutive_errors += 1
if consecutive_errors >= self._max_consecutive_errors:
self.error_occurred.emit(
self._camera_id, "Too many empty frames.\nWas the device disconnected ?"
)
break
if self._stop_event.wait(self._retry_delay):
break
continue

consecutive_errors = 0
with self._timing.measure("Single.transforms"):
frame = self._apply_worker_transforms(frame)

with self._recording_sink_lock:
recording_enabled = self._recording_enabled
recording_sink = self._recording_sink

if recording_enabled and recording_sink is not None:
try:
with self._timing.measure("Single.recording_sink"):
recording_sink(self._camera_id, frame, timestamp, timestamp_metadata)
except Exception as exc:
logger.exception(f"Failed to write frame for camera {self._camera_id}: {exc}")

with self._timing.measure("Single.emit"):
self.frame_captured.emit(self._camera_id, frame, timestamp, timestamp_metadata)

self._timing.note_frame()
self._timing.maybe_log()

except TimeoutError as exc:
self._timing.note_timeout()
self._timing.maybe_log()
if self._stop_event.is_set():
break

# In hardware-trigger mode, a timeout usually means:
# "no trigger pulse arrived during this poll interval".
# This is expected and should not count as a camera failure.
if bool(getattr(self._backend, "waits_for_hardware_trigger", False)):
self._log_trigger_wait_throttled(exc)
consecutive_errors = 0

if self._stop_event.wait(self._trigger_timeout_delay):
break # Stop event set during wait
continue

consecutive_errors += 1
if consecutive_errors >= self._max_consecutive_errors:
self.error_occurred.emit(self._camera_id, f"Camera read timeout: {exc}")
break
if self._stop_event.wait(self._retry_delay):
break
continue

except Exception as exc:
self._timing.note_error()
self._timing.maybe_log()
consecutive_errors += 1
if self._stop_event.is_set():
break
if consecutive_errors >= self._max_consecutive_errors:
self.error_occurred.emit(self._camera_id, f"Camera read error: {exc}")
break
if self._stop_event.wait(self._retry_delay):
break
continue

# Cleanup
if self._backend is not None:
try:
self._backend.close()
except Exception:
pass
self.stopped.emit(self._camera_id)

def stop(self) -> None:
self._stop_event.set()

@staticmethod
def apply_rotation(frame: np.ndarray, degrees: int) -> np.ndarray:
"""Apply rotation to frame."""
if degrees == 90:
return cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
elif degrees == 180:
return cv2.rotate(frame, cv2.ROTATE_180)
elif degrees == 270:
return cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE)
return frame

@staticmethod
def apply_crop(frame: np.ndarray, crop_region: tuple[int, int, int, int]) -> np.ndarray:
"""Apply crop to frame."""
x0, y0, x1, y1 = crop_region
height, width = frame.shape[:2]

x0 = max(0, min(x0, width))
y0 = max(0, min(y0, height))
x1 = max(x0, min(x1, width)) if x1 > 0 else width
y1 = max(y0, min(y1, height)) if y1 > 0 else height

if x0 < x1 and y0 < y1:
return frame[y0:y1, x0:x1]
return frame

def _apply_worker_transforms(self, frame: np.ndarray) -> np.ndarray:
if self._settings.rotation:
frame = self.apply_rotation(frame, self._settings.rotation)

crop_region = self._settings.get_crop_region()
if crop_region:
frame = self.apply_crop(frame, crop_region)

return frame

def _log_trigger_wait_throttled(self, exc: BaseException) -> None:
"""Log hardware-trigger wait timeouts at a controlled rate.

In trigger-waiting modes, read timeouts are expected polling misses.
Without throttling, the log can be flooded at ~10-20 messages/sec/camera.
"""
now = time.monotonic()

if now - self._last_trigger_wait_log < self._trigger_wait_log_interval:
self._trigger_wait_suppressed_count += 1
return

suppressed = self._trigger_wait_suppressed_count
self._trigger_wait_suppressed_count = 0
self._last_trigger_wait_log = now

if suppressed:
logger.debug(
"[Worker %s] waiting for hardware trigger: %s (suppressed %d repeated timeout logs)",
self._camera_id,
exc,
suppressed,
)
else:
logger.debug(
"[Worker %s] waiting for hardware trigger: %s",
self._camera_id,
exc,
)
Loading