Skip to content

Commit e9c23dc

Browse files
AlgomorphAlgomorph
committed
reworked multipair experiments for HO 3D
1 parent bc6c3bd commit e9c23dc

5 files changed

Lines changed: 560 additions & 38 deletions

File tree

‎cpp‎

‎experiment/dataset.py‎

Lines changed: 49 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -100,12 +100,26 @@ def __init__(self, calibration_file_path, first_frame_path, second_frame_path, i
100100
intrinsics=Camera.Intrinsics((480, 640), intrinsic_matrix=camera_intrinsic_matrix),
101101
depth_unit_ratio=0.001)
102102
self.rig = DepthCameraRig(cameras=(camera,))
103-
parameters = cpp_module.tsdf.Parameters2d()
104-
parameters.interpolation_method = cpp_module.tsdf.FilteringMethod.NONE
105-
parameters.projection_matrix = self.rig.depth_camera.intrinsics.intrinsic_matrix.astype(np.float32)
106-
parameters.array_offset = cpp_module.Vector2i(int(self.offset[0]), int(self.offset[2]))
107-
parameters.field_shape = cpp_module.Vector2i(self.field_size, self.field_size)
108-
self.parameters = parameters
103+
parameters_2d = cpp_module.tsdf.Parameters2d()
104+
parameters_2d.interpolation_method = cpp_module.tsdf.FilteringMethod.NONE
105+
parameters_2d.projection_matrix = self.rig.depth_camera.intrinsics.intrinsic_matrix.astype(np.float32)
106+
parameters_2d.array_offset = cpp_module.Vector2i(int(self.offset[0]), int(self.offset[2]))
107+
parameters_2d.field_shape = cpp_module.Vector2i(self.field_size, self.field_size)
108+
self.parameters_2d = parameters_2d
109+
parameters_3d = cpp_module.tsdf.Parameters3d()
110+
parameters_3d.interpolation_method = cpp_module.tsdf.FilteringMethod.NONE
111+
parameters_3d.projection_matrix = self.rig.depth_camera.intrinsics.intrinsic_matrix.astype(np.float32)
112+
parameters_3d.array_offset = cpp_module.Vector3i(int(self.offset[0]), int(self.offset[1]), int(self.offset[2]))
113+
parameters_3d.field_shape = cpp_module.Vector3i(self.field_size, self.field_size, self.field_size)
114+
self.parameters_3d = parameters_3d
115+
116+
def _generate_3d_sdf_aux_image(self, depth_image, method=cpp_module.tsdf.FilteringMethod.NONE,
117+
smoothing_coefficient=1.0):
118+
self.parameters_3d.smoothing_factor = smoothing_coefficient
119+
self.parameters_3d.interpolation_method = method
120+
generator3d = cpp_module.tsdf.Generator3d(self.parameters_3d)
121+
field = generator3d.generate(depth_image, np.identity(4, dtype=np.float32), 0)
122+
return field
109123

110124
def _generate_2d_sdf_aux_image(self, depth_image, method=cpp_module.tsdf.FilteringMethod.NONE,
111125
smoothing_coefficient=1.0, use_cpp=False):
@@ -121,18 +135,17 @@ def _generate_2d_sdf_aux_image(self, depth_image, method=cpp_module.tsdf.Filteri
121135
voxel_size=self.voxel_size,
122136
smoothing_coefficient=smoothing_coefficient)
123137
else:
124-
self.parameters.smoothing_factor = smoothing_coefficient
125-
self.parameters.interpolation_method = method
126-
generator = cpp_module.tsdf.Generator2d(self.parameters)
127-
print(depth_image.dtype)
138+
self.parameters_2d.smoothing_factor = smoothing_coefficient
139+
self.parameters_2d.interpolation_method = method
140+
generator = cpp_module.tsdf.Generator2d(self.parameters_2d)
128141
field = generator.generate(depth_image, np.identity(4, dtype=np.float32), self.image_pixel_row)
129142
return field
130143

131144
def _generate_2d_sdf_aux(self, path, method=cpp_module.tsdf.FilteringMethod.NONE,
132145
smoothing_coefficient=1.0,
133146
use_cpp=False):
134147
depth_image = cv2.imread(path, cv2.IMREAD_UNCHANGED)
135-
return self._generate_2d_sdf_aux_image(self, depth_image, method, smoothing_coefficient, use_cpp)
148+
return self._generate_2d_sdf_aux_image(depth_image, method, smoothing_coefficient, use_cpp)
136149

137150
def generate_2d_sdf_canonical(self, method=cpp_module.tsdf.FilteringMethod.NONE, smoothing_coefficient=1.0,
138151
use_cpp=False):
@@ -144,9 +157,16 @@ def generate_2d_sdf_live(self, method=cpp_module.tsdf.FilteringMethod.NONE, smoo
144157

145158
def generate_2d_sdf_fields(self, method=cpp_module.tsdf.FilteringMethod.NONE, smoothing_coefficient=1.0,
146159
use_cpp=False):
147-
live_field = self.generate_2d_sdf_live(method, smoothing_coefficient, use_cpp)
148160
canonical_field = self.generate_2d_sdf_canonical(method, smoothing_coefficient, use_cpp)
149-
return live_field, canonical_field
161+
live_field = self.generate_2d_sdf_live(method, smoothing_coefficient, use_cpp)
162+
return canonical_field, live_field
163+
164+
def generate_3d_sdf_fields(self, method=cpp_module.tsdf.FilteringMethod.NONE, smoothing_coefficient=1.0):
165+
canonical_field = self._generate_3d_sdf_aux_image(cv2.imread(self.first_frame_path, cv2.IMREAD_UNCHANGED),
166+
method, smoothing_coefficient)
167+
live_field = self._generate_3d_sdf_aux_image(cv2.imread(self.second_frame_path, cv2.IMREAD_UNCHANGED), method,
168+
smoothing_coefficient)
169+
return canonical_field, live_field
150170

151171

152172
class MaskedImageBasedFramePairDataset(ImageBasedFramePairDataset):
@@ -158,23 +178,33 @@ def __init__(self, calibration_file_path, first_frame_path, first_mask_path, sec
158178
self.first_mask_path = first_mask_path
159179
self.second_mask_path = second_mask_path
160180

161-
def _generate_2d_sdf_masked_aux(self, frame_path, mask_path, method=cpp_module.tsdf.FilteringMethod.NONE,
162-
smoothing_coefficient=1.0, use_cpp=False):
163-
depth_image = cv2.imread(frame_path, cv2.IMREAD_UNCHANGED)
181+
def _read_masked_depth_image(self, image_path, mask_path):
182+
depth_image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED)
164183
mask_image = cv2.imread(mask_path, cv2.IMREAD_UNCHANGED)
165184
max_depth = np.iinfo(np.uint16).max
166185
depth_image[mask_image == 0] = max_depth
167186
depth_image[depth_image == 0] = max_depth
187+
return depth_image
188+
189+
def _generate_2d_sdf_masked_aux(self, frame_path, mask_path, method=cpp_module.tsdf.FilteringMethod.NONE,
190+
smoothing_coefficient=1.0, use_cpp=False):
191+
depth_image = self._read_masked_depth_image(frame_path, mask_path)
168192
return super()._generate_2d_sdf_aux_image(depth_image=depth_image, method=method,
169193
smoothing_coefficient=smoothing_coefficient, use_cpp=use_cpp)
170194

195+
def _generate_3d_sdf_masked_aux(self, frame_path, mask_path, method=cpp_module.tsdf.FilteringMethod.NONE,
196+
smoothing_coefficient=1.0):
197+
depth_image = self._read_masked_depth_image(frame_path, mask_path)
198+
return super()._generate_3d_sdf_aux_image(depth_image=depth_image, method=method,
199+
smoothing_coefficient=smoothing_coefficient)
200+
171201
def generate_2d_sdf_fields(self, method=cpp_module.tsdf.FilteringMethod.NONE, smoothing_coefficient=1.0,
172202
use_cpp=False):
173-
live_field = self._generate_2d_sdf_masked_aux(self.first_frame_path, self.first_mask_path, method,
174-
smoothing_coefficient, use_cpp)
175203
canonical_field = self._generate_2d_sdf_masked_aux(self.second_frame_path, self.second_mask_path, method,
176204
smoothing_coefficient, use_cpp)
177-
return live_field, canonical_field
205+
live_field = self._generate_2d_sdf_masked_aux(self.first_frame_path, self.first_mask_path, method,
206+
smoothing_coefficient, use_cpp)
207+
return canonical_field, live_field
178208

179209

180210
datasets = {

‎experiment/hierarchical_optimizer/multipair_arguments.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919

2020
# local
2121
from ext_argparse.argument import Argument
22-
from tsdf import generation as tsdf
2322
import experiment.hierarchical_optimizer.build_helper as build_opt
2423

2524
# NB: needs to be compiled and installed / added to PYTHONPATH first
@@ -85,11 +84,14 @@ class Arguments(Enum):
8584
arg_help="Convert telemetry to videos")
8685

8786

88-
def post_process_enum_args(args):
87+
def post_process_enum_args(args, for_3d=False):
8988
Arguments.generation_method.v = \
90-
args.generation_method = tsdf.GenerationMethod.__dict__[args.generation_method]
89+
args.generation_method = cpp_module.tsdf.FilteringMethod.__dict__[args.generation_method]
9190
Arguments.implementation_language.v = args.implementation_language = \
9291
build_opt.ImplementationLanguage.__dict__[args.implementation_language]
93-
resampling_strategies = cpp_module.HierarchicalOptimizer2d.ResamplingStrategy.__dict__['values']
94-
Arguments.resampling_strategy.v = args.resampling_strategy = \
95-
[val for key, val in resampling_strategies.items() if val.name == Arguments.resampling_strategy.v][0]
92+
if for_3d:
93+
Arguments.resampling_strategy.v = cpp_module.HierarchicalOptimizer3d.ResamplingStrategy.__dict__[
94+
Arguments.resampling_strategy.v]
95+
else:
96+
Arguments.resampling_strategy.v = cpp_module.HierarchicalOptimizer2d.ResamplingStrategy.__dict__[
97+
Arguments.resampling_strategy.v]

‎run_hierarchical_optimizer2d_multipair.py‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -234,12 +234,12 @@ def main():
234234
post_process_enum_args(args)
235235
perform_optimization = not Arguments.skip_optimization.v
236236

237-
if args.generation_method == tsdf.GenerationMethod.EWA_TSDF_INCLUSIVE_CPP:
238-
generation_method_name_substring = "EWA_TI"
237+
if args.generation_method == cpp_module.tsdf.FilteringMethod.EWA_VOXEL_SPACE_INCLUSIVE:
238+
generation_method_name_substring = "EWA_VI"
239239
generation_smoothing_substring = "_sm{:03d}".format(int(Arguments.smoothing_coefficient.v * 100))
240240

241-
elif args.generation_method == tsdf.GenerationMethod.BASIC:
242-
generation_method_name_substring = "basic"
241+
elif args.generation_method == cpp_module.tsdf.FilteringMethod.NONE:
242+
generation_method_name_substring = "NONE"
243243
generation_smoothing_substring = ""
244244
else:
245245
raise ValueError("Unsupported Generation Method")
@@ -250,9 +250,9 @@ def main():
250250
else:
251251
resampling_strategy_substring = "_nearest"
252252

253-
data_subfolder = "tsdf_pairs_128_{:s}{:s}_{:02d}".format(generation_method_name_substring,
254-
generation_smoothing_substring,
255-
Arguments.dataset_number.v)
253+
data_subfolder = "tsdf_pairs_2d_128_{:s}{:s}_{:02d}".format(generation_method_name_substring,
254+
generation_smoothing_substring,
255+
Arguments.dataset_number.v)
256256
data_path = os.path.join(pu.get_reconstruction_data_directory(), "real_data/snoopy", data_subfolder)
257257

258258
# TODO: add other optimizer parameters to the name
@@ -308,7 +308,8 @@ def main():
308308

309309
for dataset in progressbar.progressbar(datasets):
310310
canonical_field, live_field = dataset.generate_2d_sdf_fields(args.generation_method,
311-
args.smoothing_coefficient)
311+
args.smoothing_coefficient,
312+
use_cpp=True)
312313
initial_fields.append((canonical_field, live_field))
313314
if args.generate_data:
314315
canonical_frame = infer_frame_number_from_filename(dataset.first_frame_path)
@@ -325,7 +326,7 @@ def main():
325326
"tsdf_frame_{:06d}.png".format(live_frame))
326327
viz.save_field(live_field, live_image_path, 1024 // dataset.field_size)
327328

328-
sys.stdout.flush()
329+
sys.stdout.flush()
329330
else:
330331
files = os.listdir(data_path)
331332
files.sort()
@@ -469,10 +470,7 @@ def main():
469470
save_bad_cases(df, out_path)
470471
save_all_cases(df, out_path)
471472

472-
print()
473-
474473
return EXIT_CODE_SUCCESS
475474

476-
477475
if __name__ == "__main__":
478476
sys.exit(main())

0 commit comments

Comments
 (0)