Skip to content

Use video-specific crop params in extract_outlier_frames #2040

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MMathisLab merged 1 commit into from
Nov 10, 2022
Merged

Use video-specific crop params in extract_outlier_frames #2040

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 16 additions & 25 deletions deeplabcut/generate_training_dataset/frame_extraction.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,9 +290,9 @@ def extract_frames(
videos = cfg.get("video_sets_original") or cfg["video_sets"]
else: #filter video_list by the ones in the config file
videos = [v for v in cfg["video_sets"] if v in videos_list]

if opencv:
from deeplabcut.utils.auxfun_videos import VideoReader
from deeplabcut.utils.auxfun_videos import VideoWriter
else:
from moviepy.editor import VideoFileClip

Expand All @@ -318,7 +318,7 @@ def extract_frames(
): # multilanguage support :)

if opencv:
cap = VideoReader(video)
cap = VideoWriter(video)
nframes = len(cap)
else:
# Moviepy:
Expand Down Expand Up @@ -355,14 +355,17 @@ def extract_frames(
except KeyError:
coords = cfg["video_sets_original"][video]["crop"].split(",")

if crop and not opencv:
clip = clip.crop(
y1=int(coords[2]),
y2=int(coords[3]),
x1=int(coords[0]),
x2=int(coords[1]),
)
elif not crop:
if crop:
if opencv:
cap.set_bbox(*map(int, coords))
else:
clip = clip.crop(
y1=int(coords[2]),
y2=int(coords[3]),
x1=int(coords[0]),
x2=int(coords[1]),
)
else:
coords = None

print("Extracting frames based on %s ..." % algo)
Expand All @@ -382,8 +385,6 @@ def extract_frames(
numframes2pick,
start,
stop,
crop,
coords,
step=cluster_step,
resizewidth=cluster_resizewidth,
color=cluster_color,
Expand Down Expand Up @@ -415,7 +416,7 @@ def extract_frames(
if opencv:
for index in frames2pick:
cap.set_to_frame(index) # extract a particular frame
frame = cap.read_frame()
frame = cap.read_frame(crop=True)
if frame is not None:
image = img_as_ubyte(frame)
img_name = (
Expand All @@ -424,17 +425,7 @@ def extract_frames(
+ str(index).zfill(indexlength)
+ ".png"
)
if crop:
io.imsave(
img_name,
image[
int(coords[2]) : int(coords[3]),
int(coords[0]) : int(coords[1]),
:,
],
) # y1 = int(coords[2]),y2 = int(coords[3]),x1 = int(coords[0]), x2 = int(coords[1]
else:
io.imsave(img_name, image)
io.imsave(img_name, image)
is_valid.append(True)
else:
print("Frame", index, " not found!")
Expand Down
27 changes: 11 additions & 16 deletions deeplabcut/refine_training_dataset/outlier_frames.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,9 @@ def ExtractFramesbasedonPreselection(
duration = clip.duration

if cfg["cropping"]: # one might want to adjust
coords = (cfg["x1"], cfg["x2"], cfg["y1"], cfg["y2"])
coords = cfg["video_sets"].get(video, {}).get("crop")
if coords is not None:
coords = list(map(int, coords.split(", ")))
else:
coords = None

Expand All @@ -655,20 +657,22 @@ def ExtractFramesbasedonPreselection(
)
elif extractionalgorithm == "kmeans":
if opencv:
if coords is not None:
vid.set_bbox(*coords)
frames2pick = frameselectiontools.KmeansbasedFrameselectioncv2(
vid,
numframes2extract,
start,
stop,
cfg["cropping"],
coords,
Index,
resizewidth=cluster_resizewidth,
color=cluster_color,
)
else:
if cfg["cropping"]:
clip = clip.crop(y1=cfg["y1"], y2=cfg["x2"], x1=cfg["x1"], x2=cfg["x2"])
if coords is not None:
clip = clip.crop(
y1=coords[2], y2=coords[3], x1=coords[0], x2=coords[1],
)
frames2pick = frameselectiontools.KmeansbasedFrameselection(
clip,
numframes2extract,
Expand All @@ -693,8 +697,6 @@ def ExtractFramesbasedonPreselection(
if opencv:
PlottingSingleFramecv2(
vid,
cfg["cropping"],
coords,
data,
bodyparts,
tmpfolder,
Expand Down Expand Up @@ -732,7 +734,7 @@ def ExtractFramesbasedonPreselection(
# Extract annotations based on DeepLabCut and store in the folder (with name derived from video name) under labeled-data
if len(frames2pick) > 0:
try:
if cfg["cropping"]:
if coords is not None:
add.add_new_videos(
config, [video], coords=[coords], copy_videos=copy_videos,
) # make sure you pass coords as a list
Expand Down Expand Up @@ -899,8 +901,6 @@ def PlottingSingleFrame(

def PlottingSingleFramecv2(
cap,
crop,
coords,
Dataframe,
bodyparts2plot,
tmpfolder,
Expand All @@ -925,16 +925,11 @@ def PlottingSingleFramecv2(
):
plt.axis("off")
cap.set_to_frame(index)
frame = cap.read_frame()
frame = cap.read_frame(crop=True)
if frame is None:
print("Frame could not be read.")
return
image = img_as_ubyte(frame)
if crop:
image = image[
int(coords[2]) : int(coords[3]), int(coords[0]) : int(coords[1]), :
]

io.imsave(imagename1, image)

if savelabeled:
Expand Down
40 changes: 4 additions & 36 deletions deeplabcut/utils/frameselectiontools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,6 @@ def KmeansbasedFrameselectioncv2(
numframes2pick,
start,
stop,
crop,
coords,
Index=None,
step=1,
resizewidth=30,
Expand Down Expand Up @@ -262,16 +260,8 @@ def KmeansbasedFrameselectioncv2(
if color:
for counter, index in tqdm(enumerate(Index)):
cap.set_to_frame(index) # extract a particular frame
frame = cap.read_frame()
frame = cap.read_frame(crop=True)
if frame is not None:
if crop:
frame = frame[
int(coords[2]) : int(coords[3]),
int(coords[0]) : int(coords[1]),
:,
]

# image=img_as_ubyte(cv2.resize(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB),None,fx=ratio,fy=ratio))
image = img_as_ubyte(
cv2.resize(
frame,
Expand All @@ -294,15 +284,8 @@ def KmeansbasedFrameselectioncv2(
else:
for counter, index in tqdm(enumerate(Index)):
cap.set_to_frame(index) # extract a particular frame
frame = cap.read_frame()
frame = cap.read_frame(crop=True)
if frame is not None:
if crop:
frame = frame[
int(coords[2]) : int(coords[3]),
int(coords[0]) : int(coords[1]),
:,
]
# image=img_as_ubyte(cv2.resize(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB),None,fx=ratio,fy=ratio))
image = img_as_ubyte(
cv2.resize(
frame,
Expand All @@ -324,16 +307,8 @@ def KmeansbasedFrameselectioncv2(
print("Extracting and downsampling...", nframes, " frames from the video.")
if color:
for counter, index in tqdm(enumerate(Index)):
frame = cap.read_frame()
frame = cap.read_frame(crop=True)
if frame is not None:
if crop:
frame = frame[
int(coords[2]) : int(coords[3]),
int(coords[0]) : int(coords[1]),
:,
]

# image=img_as_ubyte(cv2.resize(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB),None,fx=ratio,fy=ratio))
image = img_as_ubyte(
cv2.resize(
frame,
Expand All @@ -355,15 +330,8 @@ def KmeansbasedFrameselectioncv2(
)
else:
for counter, index in tqdm(enumerate(Index)):
frame = cap.read_frame()
frame = cap.read_frame(crop=True)
if frame is not None:
if crop:
frame = frame[
int(coords[2]) : int(coords[3]),
int(coords[0]) : int(coords[1]),
:,
]
# image=img_as_ubyte(cv2.resize(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB),None,fx=ratio,fy=ratio))
image = img_as_ubyte(
cv2.resize(
frame,
Expand Down