Skip to content

Commit 0218b2c

Browse files
gselzerhinerm
gselzer
authored and
hinerm
committed
Write tutorial for non-Op parallel computation
1 parent afeb0b5 commit 0218b2c

4 files changed

Lines changed: 93 additions & 31 deletions

File tree

‎imagej/imagej-ops2-tutorial/src/main/java/net/imagej/ops2/tutorial/OpParallelization.java‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,16 @@
11

22
package net.imagej.ops2.tutorial;
33

4-
import java.util.stream.Collectors;
5-
import java.util.stream.IntStream;
6-
74
import org.scijava.function.Computers;
85
import org.scijava.ops.api.OpEnvironment;
96
import org.scijava.ops.engine.DefaultOpEnvironment;
107
import org.scijava.ops.spi.OpCollection;
118
import org.scijava.ops.spi.OpMethod;
129

13-
import net.imglib2.Interval;
1410
import net.imglib2.algorithm.neighborhood.Neighborhood;
1511
import net.imglib2.algorithm.neighborhood.RectangleShape;
16-
import net.imglib2.algorithm.neighborhood.Shape;
17-
import net.imglib2.img.Img;
1812
import net.imglib2.img.array.ArrayImgs;
19-
import net.imglib2.loops.IntervalChunks;
20-
import net.imglib2.outofbounds.OutOfBoundsConstantValueFactory;
21-
import net.imglib2.parallel.Parallelization;
2213
import net.imglib2.type.numeric.integer.UnsignedByteType;
23-
import net.imglib2.view.Views;
2414

2515
/**
2616
* SciJava Ops includes a mechanism for automatically introducing concurrency to

‎imagej/imagej-ops2-tutorial/src/main/java/net/imagej/ops2/tutorial/ParallelComputation.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
2+
package net.imagej.ops2.tutorial;
3+
4+
import java.util.Arrays;
5+
import java.util.List;
6+
import java.util.function.Function;
7+
8+
import org.scijava.ops.api.OpEnvironment;
9+
import org.scijava.ops.engine.DefaultOpEnvironment;
10+
import org.scijava.types.Nil;
11+
12+
import net.imglib2.img.Img;
13+
import net.imglib2.img.array.ArrayImgs;
14+
import net.imglib2.loops.LoopBuilder;
15+
import net.imglib2.parallel.Parallelization;
16+
import net.imglib2.type.numeric.integer.UnsignedByteType;
17+
18+
/**
19+
* Using the {@link net.imglib2.parallel.Parallelization} class, we can perform
20+
* independent computations in parallel. This tutorial showcases running many
21+
* Ops in parallel using {@link net.imglib2.parallel.Parallelization}.
22+
*
23+
* @author Gabriel Selzer
24+
*/
25+
public class ParallelComputation {
26+
27+
public static void main(String... args) {
28+
OpEnvironment ops = new DefaultOpEnvironment();
29+
// To compute tasks using Parallelization, we must first gather a list of
30+
// parameters.
31+
List<Double> fillValues = Arrays.asList(1.0, 2.0, 3.0, 4.0);
32+
Img<UnsignedByteType> data = ArrayImgs.unsignedBytes(10, 10, 10);
33+
Nil<Img<UnsignedByteType>> outNil = new Nil<>() {};
34+
35+
// Note that this function will be run many times in parallel
36+
// - it's not terribly complex, but we could do much more
37+
Function<Double, Img<UnsignedByteType>> fillImage = fillValue -> {
38+
// create a new image of the same size as our data
39+
var output = ops.op("create.img").arity1().input(data).outType(outNil)
40+
.apply();
41+
// fill it with the fill value
42+
LoopBuilder.setImages(output).forEachPixel(pixel -> pixel.setReal(
43+
fillValue));
44+
// and return it
45+
return output;
46+
};
47+
48+
// Parallelization.getTaskExecutor().forEachApply() takes a list of
49+
// parameters,
50+
// and a function to apply on each parameter in the list. The function will
51+
// then be applied in parallel on each parameter, and the return is a list,
52+
// with the ith output being the application of the function on the ith
53+
// parameter.
54+
List<Img<UnsignedByteType>> filledImages = //
55+
Parallelization.getTaskExecutor().forEachApply(fillValues, fillImage);
56+
57+
for (int i = 0; i < fillValues.size(); i++) {
58+
if (filledImages.get(i) != null) {
59+
System.out.println("Image " + i + " was filled with value " +
60+
filledImages.get(i).firstElement().get());
61+
}
62+
}
63+
64+
}
65+
66+
}

‎imagej/imagej-ops2/src/test/java/net/imagej/ops2/OpRegressionTest.java‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,28 +26,31 @@
2626
* POSSIBILITY OF SUCH DAMAGE.
2727
* #L%
2828
*/
29+
2930
package net.imagej.ops2;
3031

31-
import org.junit.jupiter.api.Test;
32+
import static org.junit.jupiter.api.Assertions.assertEquals;
3233

33-
import java.util.stream.Collectors;
34-
import java.util.stream.StreamSupport;
34+
import java.util.Objects;
3535

36-
import static org.junit.jupiter.api.Assertions.assertEquals;
36+
import org.junit.jupiter.api.Assertions;
37+
import org.junit.jupiter.api.Test;
38+
import org.scijava.ops.api.OpInfo;
3739

3840
public class OpRegressionTest extends AbstractOpTest {
3941

4042
@Test
4143
public void opDiscoveryRegressionIT() {
42-
long expected = 1461;
43-
long actual = StreamSupport.stream(ops.infos().spliterator(), false).count();
44+
long expected = 1465;
45+
long actual = ops.infos().size();
4446
assertEquals(expected, actual);
4547
}
4648

47-
4849
@Test
4950
public void opDescriptionRegressionIT() {
5051
// Ensure no ops have a null description
51-
StreamSupport.stream(ops.infos().spliterator(), false).map(Object::toString).collect(Collectors.toSet());
52+
for (OpInfo info: ops.infos())
53+
Assertions.assertNotNull(info.toString(),
54+
() -> "Info from " + info.id() + " has a null description");
5255
}
5356
}

‎scijava/scijava-ops-engine/src/main/java/org/scijava/ops/engine/DefaultOpEnvironment.java‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949

5050
import org.scijava.discovery.Discoverer;
5151
import org.scijava.discovery.ManualDiscoverer;
52+
import org.scijava.function.Consumers;
5253
import org.scijava.log2.Logger;
5354
import org.scijava.log2.StderrLoggerFactory;
5455
import org.scijava.meta.Versions;
@@ -577,10 +578,11 @@ private List<RichOp<?>> resolveOpDependencies(OpCandidate candidate,
577578
@SuppressWarnings("rawtypes")
578579
private synchronized void initWrappers() {
579580
if (wrappers != null) return;
580-
wrappers = new HashMap<>();
581+
HashMap<Class<?>, OpWrapper<?>> tmp = new HashMap<>();
581582
for (Discoverer d : discoverers)
582583
for (OpWrapper wrapper : d.discover(OpWrapper.class))
583-
wrappers.put(wrapper.type(), wrapper);
584+
tmp.put(wrapper.type(), wrapper);
585+
wrappers = tmp;
584586
}
585587

586588
/**
@@ -705,15 +707,16 @@ private OpRef inferOpRef(Type type, String name, Map<TypeVariable<?>, Type> type
705707

706708
private synchronized void initOpDirectory() {
707709
if (opDirectory != null) return;
708-
opDirectory = new HashMap<>();
710+
Map<String, List<OpInfo>> tmp = new HashMap<>();
709711
// add all OpInfos that are directly discoverable
710-
discoverers.stream().flatMap(d -> d.discover(OpInfo.class).stream()).forEach(info -> addToOpIndex.accept(info, log));
712+
discoverers.stream().flatMap(d -> d.discover(OpInfo.class).stream()).forEach(info -> addToOpIndex.accept(tmp, info, log));
711713
List<OpInfoGenerator> generators = infoGenerators();
712-
discoverers.stream().flatMap(d -> d.discover(Op.class).stream()).forEach(o -> registerOpsFrom(o, generators));
713-
discoverers.stream().flatMap(d -> d.discover(OpCollection.class).stream()).forEach(o -> registerOpsFrom(o, generators));
714-
Set<OpInfo> infos = opDirectory.values().stream().flatMap(Collection::stream).map(info -> opsFromObject(info, generators)).flatMap(Collection::stream).collect(
714+
discoverers.stream().flatMap(d -> d.discover(Op.class).stream()).forEach(o -> registerOpsFrom(tmp, o, generators));
715+
discoverers.stream().flatMap(d -> d.discover(OpCollection.class).stream()).forEach(o -> registerOpsFrom(tmp, o, generators));
716+
Set<OpInfo> infos = tmp.values().stream().flatMap(Collection::stream).map(info -> opsFromObject(info, generators)).flatMap(Collection::stream).collect(
715717
Collectors.toSet());
716-
infos.forEach(info -> addToOpIndex.accept(info, log));
718+
infos.forEach(info -> addToOpIndex.accept(tmp, info, log));
719+
opDirectory = tmp;
717720
}
718721

719722
/**
@@ -729,8 +732,8 @@ private List<OpInfo> opsFromObject(Object o, List<OpInfoGenerator> generators) {
729732
.collect(Collectors.toList());
730733
}
731734

732-
private void registerOpsFrom(Object o, List<OpInfoGenerator> generators) {
733-
opsFromObject(o, generators).forEach(info -> addToOpIndex.accept(info, log));
735+
private void registerOpsFrom(final Map<String, List<OpInfo>> opDirectory, final Object o, List<OpInfoGenerator> generators) {
736+
opsFromObject(o, generators).forEach(info -> addToOpIndex.accept(opDirectory, info, log));
734737
}
735738

736739
private List<OpInfoGenerator> infoGenerators() {
@@ -749,14 +752,14 @@ private synchronized void initIdDirectory() {
749752
.forEach(info -> idDirectory.put(info.id(), info));
750753
}
751754

752-
private final BiConsumer<OpInfo, Logger> addToOpIndex = (final OpInfo opInfo, final Logger log) -> {
753-
if (opInfo.names() == null || opInfo.names().size() == 0) {
755+
private final Consumers.Arity3<Map<String, List<OpInfo>>, OpInfo, Logger> addToOpIndex = (final Map<String, List<OpInfo>> directory, final OpInfo opInfo, final Logger log) -> {
756+
if (opInfo.names() == null || opInfo.names().isEmpty()) {
754757
log.error("Skipping Op " + opInfo.implementationName() + ":\n" +
755758
"Op implementation must provide name.");
756759
return;
757760
}
758761
for (String opName : opInfo.names()) {
759-
opDirectory.computeIfAbsent(opName, name -> new ArrayList<>()).add(opInfo);
762+
directory.computeIfAbsent(opName, name -> new ArrayList<>()).add(opInfo);
760763
}
761764
};
762765

0 commit comments

Comments
 (0)