Skip to content

[SQL] Share indexes between joins when possible #5832

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mihaibudiu merged 1 commit into from
Mar 19, 2026
Merged

[SQL] Share indexes between joins when possible #5832

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions ...r/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPChainOperator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteRelNode;
import org.dbsp.sqlCompiler.compiler.visitors.VisitDecision;
import org.dbsp.sqlCompiler.compiler.visitors.inner.EquivalenceContext;
import org.dbsp.sqlCompiler.compiler.visitors.inner.Expensive;
import org.dbsp.sqlCompiler.compiler.visitors.inner.InnerVisitor;
import org.dbsp.sqlCompiler.compiler.visitors.inner.Simplify;
import org.dbsp.sqlCompiler.compiler.visitors.outer.CircuitVisitor;
Expand All @@ -29,6 +30,7 @@
import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeZSet;
import org.dbsp.util.IndentStreamBuilder;
import org.dbsp.util.Linq;
import org.dbsp.util.Maybe;
import org.dbsp.util.Utilities;

import javax.annotation.Nullable;
Expand Down Expand Up @@ -243,6 +245,90 @@ public String toString() {
builder.decrease().newline();
return builder.toString();
}

/** Compose pairs of maps that can be efficiently composed, taking into advantage
* the fact that function composition is associative. */
public DBSPChainOperator.ComputationChain shrinkMaps(DBSPCompiler compiler) {
List<DBSPChainOperator.Computation> result = new ArrayList<>();
for (DBSPChainOperator.Computation comp: this.computations()) {
if (result.isEmpty() || comp.kind() == DBSPChainOperator.ComputationKind.Filter) {
result.add(comp);
} else {
DBSPChainOperator.Computation last = Utilities.removeLast(result);
if (last.kind() == DBSPChainOperator.ComputationKind.Filter) {
result.add(last);
result.add(comp);
continue;
}

boolean expensive = Expensive.isExpensive(compiler, last.closure());
if (expensive) {
result.add(last);
} else {
DBSPClosureExpression composed;
if (last.kind() == DBSPChainOperator.ComputationKind.Map) {
composed = comp.closure().applyAfter(compiler, last.closure(), Maybe.MAYBE);
} else {
DBSPClosureExpression lastFunction = last.closure();
DBSPExpression argument = new DBSPRawTupleExpression(
lastFunction.body.field(0).borrow(),
lastFunction.body.field(1).borrow());
DBSPExpression apply = comp.closure().call(argument);
composed = apply.reduce(compiler)
.closure(lastFunction.parameters);
}
comp = new DBSPChainOperator.Computation(comp.kind(), composed);
}
result.add(comp);
}
}

if (result.size() == this.size())
return this;
return new DBSPChainOperator.ComputationChain(this.inputType(), result);
}

/** Convert Map(m1) -> Filter(f) -> Map(m2) into Filter(f \circ m1) -> Map(m2 \circ m1) if m1 is simple */
public ComputationChain shrinkMapFilterMap(DBSPCompiler compiler) {
List<DBSPChainOperator.Computation> result = new ArrayList<>();
if (this.size() < 3) {
return this;
}

// Find a sequence Map -> Filter -> Map/MapIndex
int startIndex = -1;
for (int i = 0; i < this.size() - 2; i++) {
if (this.computations().get(i).kind() == DBSPChainOperator.ComputationKind.Map &&
this.computations().get(i+1).kind() == DBSPChainOperator.ComputationKind.Filter &&
this.computations().get(i+2).kind() != DBSPChainOperator.ComputationKind.Filter) {
DBSPClosureExpression map = this.computations().get(i).closure();
if (this.computations().get(i+1).closure().shouldInlineComposition(compiler, map) &&
this.computations().get(i+2).closure().shouldInlineComposition(compiler, map)) {
startIndex = i;
break;
}
}
result.add(this.computations().get(i));
}

if (startIndex < 0)
return this;

DBSPChainOperator.Computation first = this.computations().get(startIndex);
DBSPChainOperator.Computation filter = this.computations().get(startIndex + 1);
DBSPChainOperator.Computation third = this.computations().get(startIndex + 2);

DBSPClosureExpression filterMap = filter.closure().applyAfter(compiler, first.closure(), Maybe.MAYBE);
DBSPClosureExpression mapMap = third.closure().applyAfter(compiler, first.closure(), Maybe.MAYBE);
result.add(new DBSPChainOperator.Computation(DBSPChainOperator.ComputationKind.Filter, filterMap));
result.add(new DBSPChainOperator.Computation(third.kind(), mapMap));
// Keep the subsequent unchanged
for (int i = startIndex + 3; i < this.size(); i++) {
result.add(this.computations().get(i));
}

return new DBSPChainOperator.ComputationChain(this.inputType(), result);
}
}

@Override
Expand Down
2 changes: 2 additions & 0 deletions ...compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustInnerVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2326,6 +2326,8 @@ public VisitDecision preorder(DBSPFieldExpression expression) {
this.builder.append(expression.fieldNo);
if (fieldTypeIsNullable && !expression.getType().hasCopy() && !avoidRef) {
this.builder.append(".as_ref()");
} else if (fieldTypeIsNullable) {
this.builder.append(".clone()");
}
this.builder.append(")").decrease();
}
Expand Down
2 changes: 1 addition & 1 deletion ...SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -936,7 +936,7 @@ public DBSPExpression visitCall(RexCall call) {
DBSPType type = this.typeCompiler.convertType(node.getPositionRange(), call.getType(), false);
// If type is NULL we can skip the call altogether...
if (type.is(DBSPTypeNull.class))
return DBSPNullLiteral.INSTANCE;
return new DBSPNullLiteral();
Utilities.enforce(!type.is(DBSPTypeStruct.class));

final RexCall finalCall = call;
Expand Down
1 change: 0 additions & 1 deletion ...er/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/aggregates/AggregateCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -520,7 +520,6 @@ void processBasic(SqlBasicAggFunction function) {
final DBSPExpression increment = this.incrementOperation(
node, opcode, accumulatorType, accumulator, aggregatedValue, this.filterArgument());
final DBSPTypeUser semigroup = new DBSPTypeUser(node, SEMIGROUP, semigroupName, false, accumulatorType);

var acc2 = accumulatorType.var();
DBSPClosureExpression postProcessing = ExpressionCompiler.expandTuple(node, acc2.field(1))
.closure(acc2);
Expand Down
6 changes: 6 additions & 0 deletions ...mpiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/EquivalenceContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import org.dbsp.sqlCompiler.ir.IDBSPDeclaration;
import org.dbsp.sqlCompiler.ir.expression.DBSPExpression;

import javax.annotation.CheckReturnValue;
import javax.annotation.Nullable;
import java.util.List;

Expand Down Expand Up @@ -31,10 +32,12 @@ public EquivalenceContext() {
this.leftToRight = new Substitution<>();
}

@CheckReturnValue
public static boolean equiv(@Nullable DBSPExpression left, @Nullable DBSPExpression right) {
return new EquivalenceContext().equivalent(left, right);
}

@CheckReturnValue
public static boolean equiv(@Nullable DBSPAggregateList left, @Nullable DBSPAggregateList right) {
if (left == null)
return right == null;
Expand All @@ -43,6 +46,7 @@ public static boolean equiv(@Nullable DBSPAggregateList left, @Nullable DBSPAggr
return left.equivalent(right);
}

@CheckReturnValue
public boolean equivalent(@Nullable DBSPExpression left, @Nullable DBSPExpression right) {
if (left == null)
return right == null;
Expand All @@ -51,6 +55,7 @@ public boolean equivalent(@Nullable DBSPExpression left, @Nullable DBSPExpressio
return left.equivalent(this, right);
}

@CheckReturnValue
public boolean equivalent(@Nullable DBSPExpression[] left, @Nullable DBSPExpression[] right) {
if (left == null)
return right == null;
Expand All @@ -64,6 +69,7 @@ public boolean equivalent(@Nullable DBSPExpression[] left, @Nullable DBSPExpress
return true;
}

@CheckReturnValue
public <T extends DBSPExpression> boolean equivalent(@Nullable List<T> left, @Nullable List<T> right) {
if (left == null)
return right == null;
Expand Down
7 changes: 7 additions & 0 deletions ...er/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Expensive.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import org.dbsp.sqlCompiler.ir.IDBSPInnerNode;
import org.dbsp.sqlCompiler.ir.expression.DBSPApplyExpression;
import org.dbsp.sqlCompiler.ir.expression.DBSPApplyMethodExpression;
import org.dbsp.sqlCompiler.ir.expression.DBSPExpression;
import org.dbsp.sqlCompiler.ir.type.DBSPType;

/** Visitor which detects whether an expression contains "expensive" subexpressions.
Expand Down Expand Up @@ -64,4 +65,10 @@ public VisitDecision preorder(DBSPApplyMethodExpression unused) {
this.expensive = true;
return VisitDecision.STOP;
}

public static boolean isExpensive(DBSPCompiler compiler, DBSPExpression expression) {
Expensive expensive = new Expensive(compiler);
expensive.apply(expression);
return expensive.isExpensive();
}
}
4 changes: 2 additions & 2 deletions ...piler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/InnerRewriteVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -648,7 +648,7 @@ public VisitDecision preorder(DBSPKeywordLiteral expression) {
public VisitDecision preorder(DBSPNullLiteral expression) {
this.push(expression);
this.pop(expression);
DBSPExpression result = DBSPNullLiteral.INSTANCE;
DBSPExpression result = new DBSPNullLiteral();
this.map(expression, result);
return VisitDecision.STOP;
}
Expand All @@ -657,7 +657,7 @@ public VisitDecision preorder(DBSPNullLiteral expression) {
public VisitDecision preorder(DBSPVoidLiteral expression) {
this.push(expression);
this.pop(expression);
DBSPExpression result = DBSPVoidLiteral.INSTANCE;
DBSPExpression result = new DBSPVoidLiteral();
this.map(expression, result);
return VisitDecision.STOP;
}
Expand Down
6 changes: 3 additions & 3 deletions ...ler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Simplify.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,9 +458,9 @@ public void postorder(DBSPFieldExpression expression) {
block.lastExpression.field(expression.fieldNo));
} else if (source.is(DBSPIfExpression.class)) {
DBSPIfExpression conditional = source.to(DBSPIfExpression.class);
DBSPExpression negative = conditional.negative != null ? conditional.negative.field(expression.fieldNo) : null;
result = new DBSPIfExpression(source.getNode(), conditional.condition,
conditional.positive.field(expression.fieldNo),
conditional.negative != null ? conditional.negative.field(expression.fieldNo) : null);
conditional.positive.field(expression.fieldNo), negative);
} else if (source.is(DBSPCloneExpression.class)) {
result = new DBSPFieldExpression(expression.getNode(),
source.to(DBSPCloneExpression.class).expression,
Expand Down Expand Up @@ -520,7 +520,7 @@ public void postorder(DBSPIfExpression expression) {
} else {
result = negative;
if (result == null)
result = DBSPVoidLiteral.INSTANCE;
result = new DBSPVoidLiteral();
}
}
} else if (negative != null &&
Expand Down
3 changes: 2 additions & 1 deletion ...compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,10 +108,11 @@ void createOptimizer() {
this.add(new RemoveIAfterD(compiler));
this.add(new DeadCode(compiler, true));
this.add(new Simplify(compiler).circuitRewriter(true));
this.add(new RemoveFilters(compiler));
this.add(new RemoveConstantFilters(compiler));
this.add(new OptimizeWithGraph(compiler, g -> new OptimizeProjectionVisitor(compiler, g)));
this.add(new OptimizeWithGraph(compiler,
g -> new OptimizeProjections(compiler, true, g, operatorsAnalyzed)));
this.add(new ShareIndexes(compiler));
// Combining Joins with subsequent filters can improve the precision of the monotonicity analysis
this.add(new OptimizeWithGraph(compiler, g -> new FilterJoinVisitor(compiler, g)));
this.add(new MonotoneAnalyzer(compiler));
Expand Down
4 changes: 1 addition & 3 deletions .../src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CloneOperatorsWithFanout.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,7 @@ boolean shouldCloneInput(DBSPUnaryOperator operator) {
input.is(DBSPFilterOperator.class)))
return false;
DBSPClosureExpression function = input.to(DBSPSimpleOperator.class).getClosureFunction();
Expensive expensive = new Expensive(this.compiler);
expensive.apply(function);
return !expensive.isExpensive();
return !Expensive.isExpensive(this.compiler, function);
}

void cloneInput(DBSPUnaryOperator operator) {
Expand Down
97 changes: 2 additions & 95 deletions ...-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ImplementChains.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,113 +7,20 @@
import org.dbsp.sqlCompiler.circuit.operator.DBSPMapOperator;
import org.dbsp.sqlCompiler.circuit.operator.DBSPSimpleOperator;
import org.dbsp.sqlCompiler.compiler.DBSPCompiler;
import org.dbsp.sqlCompiler.compiler.visitors.inner.Expensive;
import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression;
import org.dbsp.sqlCompiler.ir.expression.DBSPExpression;
import org.dbsp.sqlCompiler.ir.expression.DBSPRawTupleExpression;
import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeZSet;
import org.dbsp.util.Maybe;
import org.dbsp.util.Utilities;

import java.util.ArrayList;
import java.util.List;

/** Implement {@link org.dbsp.sqlCompiler.circuit.operator.DBSPChainOperator} */
public class ImplementChains extends CircuitCloneVisitor {
public ImplementChains(DBSPCompiler compiler) {
super(compiler, false);
}

/** Compose pairs of maps that can be efficiently composed, taking into advantage
* the fact that function composition is associative. */
DBSPChainOperator.ComputationChain shrinkMaps(DBSPChainOperator.ComputationChain chain) {
List<DBSPChainOperator.Computation> result = new ArrayList<>();
for (DBSPChainOperator.Computation comp: chain.computations()) {
if (result.isEmpty() || comp.kind() == DBSPChainOperator.ComputationKind.Filter) {
result.add(comp);
} else {
DBSPChainOperator.Computation last = Utilities.removeLast(result);
if (last.kind() == DBSPChainOperator.ComputationKind.Filter) {
result.add(last);
result.add(comp);
continue;
}

Expensive expensive = new Expensive(compiler);
expensive.apply(last.closure());
if (expensive.isExpensive()) {
result.add(last);
} else {
DBSPClosureExpression composed;
if (last.kind() == DBSPChainOperator.ComputationKind.Map) {
composed = comp.closure().applyAfter(compiler, last.closure(), Maybe.MAYBE);
} else {
DBSPClosureExpression lastFunction = last.closure();
DBSPExpression argument = new DBSPRawTupleExpression(
lastFunction.body.field(0).borrow(),
lastFunction.body.field(1).borrow());
DBSPExpression apply = comp.closure().call(argument);
composed = apply.reduce(this.compiler())
.closure(lastFunction.parameters);
}
comp = new DBSPChainOperator.Computation(comp.kind(), composed);
}
result.add(comp);
}
}

if (result.size() == chain.size())
return chain;
return new DBSPChainOperator.ComputationChain(chain.inputType(), result);
}

/** Convert Map(m1) -> Filter(f) -> Map(m2) into Filter(f \circ m1) -> Map(m2 \circ m1) if m1 is simple */
DBSPChainOperator.ComputationChain shrinkMapFilterMap(DBSPChainOperator.ComputationChain chain) {
List<DBSPChainOperator.Computation> result = new ArrayList<>();
if (chain.size() < 3) {
return chain;
}

// Find a sequence Map -> Filter -> Map/MapIndex
int startIndex = -1;
for (int i = 0; i < chain.size() - 2; i++) {
if (chain.computations().get(i).kind() == DBSPChainOperator.ComputationKind.Map &&
chain.computations().get(i+1).kind() == DBSPChainOperator.ComputationKind.Filter &&
chain.computations().get(i+2).kind() != DBSPChainOperator.ComputationKind.Filter) {
DBSPClosureExpression map = chain.computations().get(i).closure();
if (chain.computations().get(i+1).closure().shouldInlineComposition(this.compiler, map) &&
chain.computations().get(i+2).closure().shouldInlineComposition(this.compiler, map)) {
startIndex = i;
break;
}
}
result.add(chain.computations().get(i));
}

if (startIndex < 0)
return chain;

DBSPChainOperator.Computation first = chain.computations().get(startIndex);
DBSPChainOperator.Computation filter = chain.computations().get(startIndex + 1);
DBSPChainOperator.Computation third = chain.computations().get(startIndex + 2);

DBSPClosureExpression filterMap = filter.closure().applyAfter(compiler, first.closure(), Maybe.MAYBE);
DBSPClosureExpression mapMap = third.closure().applyAfter(compiler, first.closure(), Maybe.MAYBE);
result.add(new DBSPChainOperator.Computation(DBSPChainOperator.ComputationKind.Filter, filterMap));
result.add(new DBSPChainOperator.Computation(third.kind(), mapMap));
// Keep the subsequent unchanged
for (int i = startIndex + 3; i < chain.size(); i++) {
result.add(chain.computations().get(i));
}

return new DBSPChainOperator.ComputationChain(chain.inputType(), result);
}

@Override
public void postorder(DBSPChainOperator node) {
DBSPChainOperator.ComputationChain chain = this.shrinkMaps(node.chain);
DBSPChainOperator.ComputationChain chain = node.chain.shrinkMaps(this.compiler);
while (true) {
var newChain = this.shrinkMapFilterMap(chain);
var newChain = chain.shrinkMapFilterMap(this.compiler);
if (newChain == chain)
break;
chain = newChain;
Expand Down
6 changes: 3 additions & 3 deletions ...src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeProjectionVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@
* - constant followed by projection
* - flatmap followed by projection
* - join followed by projection
* - join followed by mapindex projection
* - indexjoin followed by mapindex projection
* - indexjoin followed by map projection
* - join followed by mapIndex projection
* - indexJoin followed by mapIndex projection
* - indexJoin followed by map projection
* Projections are map operations that have a function with a very simple
* structure. The function is analyzed using the 'Projection' visitor. */
public class OptimizeProjectionVisitor extends CircuitCloneWithGraphsVisitor {
Expand Down
Loading
Loading