Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
213 changes: 123 additions & 90 deletions deeplabcut/utils/visualization.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,12 @@
#
# Licensed under GNU Lesser General Public License v3.0
#
"""
DeepLabCut2.0 Toolbox (deeplabcut.org)
© A. & M. Mathis Labs
https://github.com/DeepLabCut/DeepLabCut
Please see AUTHORS for contributors.

https://github.com/DeepLabCut/DeepLabCut/blob/master/AUTHORS
Licensed under GNU Lesser General Public License v3.0
"""

from __future__ import annotations

import os
from pathlib import Path
from typing import Literal

import matplotlib.patches as patches
import matplotlib.pyplot as plt
Expand All @@ -34,6 +26,9 @@

from deeplabcut.utils import auxfun_videos, auxiliaryfunctions

PlotMode = Literal["bodypart", "individual"]
BoundingBoxColor = Colormap | str | None


def get_cmap(n: int, name: str = "hsv") -> Colormap:
"""Get the cmap.
Expand Down Expand Up @@ -71,13 +66,10 @@ def make_labeled_image(
dotsize = cfg["dotsize"] # =15

if ax is None:
if np.ndim(frame) > 2: # color image!
h, w, numcolors = np.shape(frame)
else:
h, w = np.shape(frame)
h, w = np.shape(frame)[:2]
_, ax = prepare_figure_axes(w, h, scaling)
ax.imshow(frame, "gray")
for _scorerindex, loopscorer in enumerate(Scorers):
for loopscorer in Scorers:
for bpindex, bp in enumerate(bodyparts):
if np.isfinite(
DataCombined[loopscorer][bp]["y"].iloc[imagenr] + DataCombined[loopscorer][bp]["x"].iloc[imagenr]
Expand Down Expand Up @@ -132,6 +124,7 @@ def make_multianimal_labeled_image(
bounding_boxes: tuple[np.ndarray, np.ndarray] | None = None,
bboxes_cutoff: float = 0.6,
bboxes_color: Colormap | str | None = None,
color_offset: int = 0,
) -> plt.Axes:
"""Plots groundtruth labels and predictions onto the matplotlib's axes, with the
specified graphical parameters.
Expand All @@ -154,14 +147,15 @@ def make_multianimal_labeled_image(
If Colormap is passed -> each bounding box will be colored into its own color from the colormap.
If string is passed -> all bboxes will be of string's defined color.
If None -> all bboxes will be colored into a default color.
color_offset: Index offset applied when selecting colors from the colormap.

Returns:
matplotlib Axes object with plotted labels and predictions.
"""
if labels is None:
labels = ["+", ".", "x"]
if ax is None:
h, w, _ = np.shape(frame)
h, w = frame.shape[:2]
_, ax = prepare_figure_axes(w, h)
ax.imshow(frame, "gray")

Expand All @@ -187,7 +181,7 @@ def make_multianimal_labeled_image(
ax.add_patch(rectangle)

for n, data in enumerate(zip(coords_truth, coords_pred, probs_pred, strict=False)):
color = colors(n)
color = colors(n + color_offset)
coord_gt, coord_pred, prob_pred = data

ax.plot(*coord_gt.T, labels[0], ms=dotsize, alpha=alphavalue, color=color)
Expand Down Expand Up @@ -261,6 +255,11 @@ def save_labeled_frame(
dest_folder: Path,
belongs_to_train: bool,
) -> None:
"""Save the labeled frame to disk.

Note: folder creation is handled upstream.
This function assumes that the destination folder already exists.
"""
imagename = image_path.parts[-1]
imfoldername = image_path.parts[-2]
if belongs_to_train:
Expand Down Expand Up @@ -351,7 +350,7 @@ def make_labeled_images_from_dataframe(
cmap = get_cmap(nindividuals, cfg["colormap"])
colors = cmap(map_)
except KeyError as e:
raise Exception("Coloring by individuals is only valid for multi-animal data") from e
raise ValueError("Coloring by individuals requires an 'individuals' column level") from e
else:
raise ValueError("`color_by` must be either `bodypart` or `individual`.")

Expand Down Expand Up @@ -445,14 +444,14 @@ def plot_evaluation_results(
output_folder: Path,
in_train_set: bool,
plot_unique_bodyparts: bool = False,
mode: str = "bodypart",
mode: PlotMode = "bodypart",
colormap: str = "rainbow",
dot_size: int = 12,
alpha_value: float = 0.7,
p_cutoff: float = 0.6,
bounding_boxes: dict | None = None,
bboxes_cutoff: float = 0.6,
bounding_boxes_color: str = "auto",
bounding_boxes_color: BoundingBoxColor = "auto",
) -> None:
"""Creates labeled images using the results of inference, and saves them to an
output folder.
Expand Down Expand Up @@ -485,7 +484,11 @@ def plot_evaluation_results(
if bounding_boxes is None:
bounding_boxes = {}

if mode not in {"bodypart", "individual"}:
raise ValueError(f"Invalid mode: {mode}. Must be one of 'bodypart' or 'individual'.")

for row_index, row in df_combined.iterrows():
plot_unique_for_row = plot_unique_bodyparts
if isinstance(row_index, str):
image_rel_path = Path(row_index)
data_folder = image_rel_path.parent.parent.name
Expand All @@ -497,12 +500,34 @@ def plot_evaluation_results(
image_path = project_root / data_folder / video / image
frame = auxfun_videos.imread(str(image_path), mode="skimage")

row_multi = row.loc[(slice(None), row.index.get_level_values("individuals") != "single")]
individuals = len(row_multi.index.get_level_values("individuals").unique())
bodyparts = len(row_multi.index.get_level_values("bodyparts").unique())
row_multi = row.loc[row.index.get_level_values("individuals") != "single"]

df_gt = row_multi[scorer]
df_predictions = row_multi[model_name]

gt_individuals = df_gt.index.get_level_values("individuals").unique()
pred_individuals = df_predictions.index.get_level_values("individuals").unique()

gt_bodyparts = df_gt.index.get_level_values("bodyparts").unique()
pred_bodyparts = df_predictions.index.get_level_values("bodyparts").unique()

if len(gt_individuals) != len(pred_individuals):
print(f"Warning: Individual count mismatch for {image}")
print(f" Ground truth individual count: {len(gt_individuals)}")
print(f" Predictions individual count: {len(pred_individuals)}")
print(" Skipping visualization for this image")
continue

if list(gt_bodyparts) != list(pred_bodyparts): # keep ordering of bodyparts
print(f"Warning: Bodypart mismatch for {image}")
print(f" Ground truth: {list(gt_bodyparts)}")
print(f" Predictions: {list(pred_bodyparts)}")
print(" Skipping visualization for this image")
continue

individuals = len(gt_individuals)
bodyparts = len(gt_bodyparts)

# Shape (num_individuals, num_bodyparts, xy)
try:
ground_truth = df_gt.to_numpy().reshape((individuals, bodyparts, 2))
Expand All @@ -515,92 +540,100 @@ def plot_evaluation_results(
expected_size_pred = individuals * bodyparts * 3

print(f"Warning: DataFrame reshape failed for {image}")
print(f" Expected: {individuals} individuals, {bodyparts} bodyparts")
print(f" Expected: {individuals} individual(s), {bodyparts} bodypart(s)")
print(f" Ground truth: {actual_size_gt} elements (expected {expected_size_gt})")
print(f" Predictions: {actual_size_pred} elements (expected {expected_size_pred})")
print(" Skipping visualization for this image")
continue

bboxes = bounding_boxes.get(row_index)

if plot_unique_bodyparts:
row_unique = row.loc[(slice(None), row.index.get_level_values("individuals") == "single")]
unique_individuals = 1
unique_bodyparts = len(row_unique.index.get_level_values("bodyparts").unique())
try:
unique_ground_truth = row_unique[scorer].to_numpy().reshape((unique_individuals, unique_bodyparts, 2))
unique_predictions = (
row_unique[model_name].to_numpy().reshape((unique_individuals, unique_bodyparts, 3))
)
except ValueError:
# Handle cases where unique bodyparts reshape fails
print(f"Warning: Unique bodyparts reshape failed for {image}, skipping unique bodyparts")
plot_unique_bodyparts = False
if plot_unique_for_row:
row_unique = row.loc[row.index.get_level_values("individuals") == "single"]
if row_unique.empty:
plot_unique_for_row = False
else:
unique_gt = row_unique[scorer]
unique_pred = row_unique[model_name]

gt_unique_bodyparts = unique_gt.index.get_level_values("bodyparts").unique()
pred_unique_bodyparts = unique_pred.index.get_level_values("bodyparts").unique()

if list(gt_unique_bodyparts) != list(pred_unique_bodyparts):
print(f"Warning: Unique bodypart mismatch for {image}, skipping unique bodyparts")
plot_unique_for_row = False
else:
unique_bodyparts = len(gt_unique_bodyparts)

try:
unique_ground_truth = unique_gt.to_numpy().reshape((1, unique_bodyparts, 2))
unique_predictions = unique_pred.to_numpy().reshape((1, unique_bodyparts, 3))
except ValueError:
# Handle cases where unique bodyparts reshape fails
print(f"Warning: Unique bodyparts reshape failed for {image}, skipping unique bodyparts")
plot_unique_for_row = False

fig, ax = create_minimal_figure()
h, w, _ = np.shape(frame)
fig.set_size_inches(w / 100, h / 100)
ax.set_xlim(0, w)
ax.set_ylim(0, h)
ax.invert_yaxis()

if mode == "bodypart":
num_colors = bodyparts
if plot_unique_bodyparts:
num_colors += unique_bodyparts

colors = get_cmap(num_colors, name=colormap)
predictions = predictions.swapaxes(0, 1)
ground_truth = ground_truth.swapaxes(0, 1)
elif mode == "individual":
colors = get_cmap(individuals + 1, name=colormap)
else:
colors = []
try:
h, w = frame.shape[:2]
fig.set_size_inches(w / 100, h / 100)
ax.set_xlim(0, w)
ax.set_ylim(0, h)
ax.invert_yaxis()

if bounding_boxes_color == "auto":
if mode == "bodypart":
bboxes_color = None
elif mode == "individual":
bboxes_color = get_cmap(individuals + 1, name=colormap)
num_colors = bodyparts
if plot_unique_for_row:
num_colors += unique_bodyparts

colors = get_cmap(num_colors, name=colormap)
predictions = predictions.swapaxes(0, 1)
ground_truth = ground_truth.swapaxes(0, 1)
else:
raise ValueError(f"Invalid mode: {mode}")
else:
bboxes_color = bounding_boxes_color

ax = make_multianimal_labeled_image(
frame=frame,
coords_truth=ground_truth,
coords_pred=predictions[:, :, :2],
probs_pred=predictions[:, :, 2:],
colors=colors,
dotsize=dot_size,
alphavalue=alpha_value,
pcutoff=p_cutoff,
ax=ax,
bounding_boxes=bboxes,
bboxes_cutoff=bboxes_cutoff,
bboxes_color=bboxes_color,
)
if plot_unique_bodyparts:
unique_predictions = unique_predictions.swapaxes(0, 1)
unique_ground_truth = unique_ground_truth.swapaxes(0, 1)
colors = get_cmap(individuals + 1, name=colormap)

if bounding_boxes_color == "auto":
bboxes_color = None if mode == "bodypart" else get_cmap(individuals + 1, name=colormap)
else:
bboxes_color = bounding_boxes_color

ax = make_multianimal_labeled_image(
frame=frame,
coords_truth=unique_ground_truth,
coords_pred=unique_predictions[:, :, :2],
probs_pred=unique_predictions[:, :, 2:],
coords_truth=ground_truth,
coords_pred=predictions[:, :, :2],
probs_pred=predictions[:, :, 2:],
colors=colors,
dotsize=dot_size,
alphavalue=alpha_value,
pcutoff=p_cutoff,
ax=ax,
bounding_boxes=bboxes,
bboxes_cutoff=bboxes_cutoff,
bboxes_color=bboxes_color,
)
if plot_unique_for_row:
if mode == "bodypart":
unique_predictions = unique_predictions.swapaxes(0, 1)
unique_ground_truth = unique_ground_truth.swapaxes(0, 1)
ax = make_multianimal_labeled_image(
Comment thread
C-Achard marked this conversation as resolved.
frame=frame,
coords_truth=unique_ground_truth,
coords_pred=unique_predictions[:, :, :2],
probs_pred=unique_predictions[:, :, 2:],
colors=colors,
color_offset=bodyparts if mode == "bodypart" else individuals,
dotsize=dot_size,
alphavalue=alpha_value,
pcutoff=p_cutoff,
ax=ax,
)

save_labeled_frame(
fig,
image_path,
output_folder,
belongs_to_train=in_train_set,
)
erase_artists(ax)
plt.close()
save_labeled_frame(
fig,
image_path,
output_folder,
belongs_to_train=in_train_set,
)
erase_artists(ax)
finally:
plt.close(fig)
Loading
Loading