Skip to content

[SQL] Improved unused field analysis: proper dataflow analysis, and less conservative handling of nullable tuples #5826

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mihaibudiu merged 1 commit into from
Mar 19, 2026
Merged

[SQL] Improved unused field analysis: proper dataflow analysis, and less conservative handling of nullable tuples #5826

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions ...-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@
import org.dbsp.sqlCompiler.compiler.frontend.statements.TableModifyStatement;
import org.dbsp.sqlCompiler.compiler.visitors.inner.Simplify;
import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.FieldUseMap;
import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.FindUnusedFields;
import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.FindUsedFields;
import org.dbsp.sqlCompiler.ir.DBSPParameter;
import org.dbsp.sqlCompiler.ir.aggregate.DBSPFold;
import org.dbsp.sqlCompiler.ir.aggregate.IAggregate;
Expand Down Expand Up @@ -1070,8 +1070,8 @@ DBSPSimpleOperator castOutput(CalciteRelNode node, DBSPSimpleOperator operator,
DBSPType inputElementType = operator.getOutputZSetElementType();
if (inputElementType.sameType(outputElementType))
return operator;
DBSPExpression function = inputElementType.caster(outputElementType, DBSPCastExpression.CastType.SqlUnsafe)
.reduce(this.compiler);
DBSPClosureExpression caster = inputElementType.caster(outputElementType, DBSPCastExpression.CastType.SqlUnsafe);
DBSPExpression function = caster.reduce(this.compiler);
DBSPSimpleOperator map = new DBSPMapOperator(
node, function, TypeCompiler.makeZSet(outputElementType), operator.outputPort());
this.addOperator(map);
Expand Down Expand Up @@ -1557,7 +1557,7 @@ private void visitJoin(LogicalJoin join) {
DBSPExpression result = makeAnd(pullLeft);
DBSPClosureExpression clo = result.closure(t);
leftPulled = new DBSPFilterOperator(node, clo, left.outputPort());
leftPulledFields = FindUnusedFields.computeUsedFields(clo, this.compiler);
leftPulledFields = FindUsedFields.computeUsedFields(clo, this.compiler);
this.addOperator(leftPulled);
}
if (!decomposition.rightPredicates.isEmpty()) {
Expand All @@ -1568,7 +1568,7 @@ private void visitJoin(LogicalJoin join) {
DBSPExpression result = makeAnd(pullRight);
DBSPClosureExpression clo = result.closure(t);
rightPulled = new DBSPFilterOperator(node, clo, right.outputPort());
rightPulledFields = FindUnusedFields.computeUsedFields(clo, this.compiler);
rightPulledFields = FindUsedFields.computeUsedFields(clo, this.compiler);
this.addOperator(rightPulled);
}

Expand Down Expand Up @@ -1735,7 +1735,7 @@ private void visitJoin(LogicalJoin join) {
this.addOperator(joinResult);
inner = new DBSPFilterOperator(node, postJoinCondition, joinResult.outputPort());
joinResult = inner;
conditionFields = FindUnusedFields.computeUsedFields(postJoinCondition, this.compiler);
conditionFields = FindUsedFields.computeUsedFields(postJoinCondition, this.compiler);
}
}

Expand Down
4 changes: 4 additions & 0 deletions ...SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,10 @@ public DBSPExpression inputIndex(CalciteObject node, int index) {
DBSPTypeTuple type = this.inputRow.getType().deref().to(DBSPTypeTuple.class);
if (index < type.size()) {
DBSPExpression field = this.inputRow.deref().field(index);
if (field.getType().is(DBSPTypeNull.class)) {
// Optimize away field accesses in ROW members which have null types.
return field.getType().none();
}
return field.applyCloneIfNeeded();
}
if (index - type.size() < this.constants.size())
Expand Down
16 changes: 9 additions & 7 deletions ...er/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/aggregates/AggregateCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -509,19 +509,21 @@ void processBasic(SqlBasicAggFunction function) {
Linq.list(
dataType.getFieldType(1).withMayBeNull(true),
this.resultType));
DBSPExpression zero = new DBSPRawTupleExpression(
final DBSPExpression zero = new DBSPRawTupleExpression(
accumulatorType.tupFields[0].none(),
accumulatorType.tupFields[1].defaultValue());
DBSPExpression aggregatedValue =
final DBSPExpression aggregatedValue =
new DBSPRawTupleExpression(
ExpressionCompiler.expandTuple(node, tuple.fields[1]),
ExpressionCompiler.expandTuple(node, tuple.fields[0]));
DBSPVariablePath accumulator = accumulatorType.var();
DBSPExpression increment = this.incrementOperation(
final DBSPVariablePath accumulator = accumulatorType.var();
final DBSPExpression increment = this.incrementOperation(
node, opcode, accumulatorType, accumulator, aggregatedValue, this.filterArgument());
DBSPTypeUser semigroup = new DBSPTypeUser(node, SEMIGROUP, semigroupName, false, accumulatorType);
DBSPClosureExpression postProcessing = ExpressionCompiler.expandTuple(node, accumulator.field(1))
.closure(accumulator);
final DBSPTypeUser semigroup = new DBSPTypeUser(node, SEMIGROUP, semigroupName, false, accumulatorType);

var acc2 = accumulatorType.var();
DBSPClosureExpression postProcessing = ExpressionCompiler.expandTuple(node, acc2.field(1))
.closure(acc2);

if (this.filterArgument >= 0) {
aggregate = new NonLinearAggregate(
Expand Down
2 changes: 2 additions & 0 deletions ...ler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Simplify.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,8 @@ public void postorder(DBSPFieldExpression expression) {
DBSPExpression result = source.field(expression.fieldNo);
if (source.is(DBSPBaseTupleExpression.class)) {
result = source.to(DBSPBaseTupleExpression.class).get(expression.fieldNo);
if (source.getType().mayBeNull && !result.getType().mayBeNull)
result = result.some();
} if (source.is(DBSPBlockExpression.class)) {
DBSPBlockExpression block = source.to(DBSPBlockExpression.class);
Utilities.enforce(block.lastExpression != null);
Expand Down
2 changes: 1 addition & 1 deletion ...compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ void createOptimizer() {

this.add(new OptimizeWithGraph(compiler, g -> new CloneOperatorsWithFanout(compiler, g)));
this.add(new LinearPostprocessRetainKeys(compiler));
this.add(new IndexedInputs(compiler));
this.add(new ExpandIndexedInputs(compiler));
this.add(new OptimizeWithGraph(compiler, g -> new FilterJoinVisitor(compiler, g)));
this.add(new DeadCode(compiler, true));
this.add(new Simplify(compiler).circuitRewriter(true));
Expand Down
94 changes: 94 additions & 0 deletions ...piler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ExpandIndexedInputs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package org.dbsp.sqlCompiler.compiler.visitors.outer;

import org.dbsp.sqlCompiler.circuit.operator.DBSPDeindexOperator;
import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceMultisetOperator;
import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceMapOperator;
import org.dbsp.sqlCompiler.compiler.DBSPCompiler;
import org.dbsp.sqlCompiler.compiler.InputColumnMetadata;
import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.FindCommonProjections;
import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.ReplaceCommonProjections;
import org.dbsp.sqlCompiler.ir.type.DBSPType;
import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeIndexedZSet;
import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple;
import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeZSet;

import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;

/** Invokes {@link IndexedInputs} and then optimizes the circuit a bit */
public class ExpandIndexedInputs extends Passes {
public ExpandIndexedInputs(DBSPCompiler compiler) {
super("ExpandIndexedInputs", compiler);
this.add(new IndexedInputs(compiler));
Graph graph = new Graph(compiler);
this.add(graph);
FindCommonProjections fcp = new FindCommonProjections(compiler, graph.getGraphs());
this.add(fcp);
this.add(new ReplaceCommonProjections(compiler, fcp));
}

/**
* Given a source node, return the type of the indexed Z-set that has as keys
* the key fields, and as value the output value. Return null if there are no key fields.
*/
@Nullable
public static DBSPTypeIndexedZSet getIndexedType(DBSPSourceMultisetOperator node) {
List<DBSPType> keyFields = new ArrayList<>();
List<Integer> keyColumnFields = new ArrayList<>();
int i = 0;
for (InputColumnMetadata inputColumnMetadata : node.metadata.getColumns()) {
if (inputColumnMetadata.isPrimaryKey) {
keyColumnFields.add(i);
keyFields.add(inputColumnMetadata.type);
}
i++;
}
if (keyColumnFields.isEmpty()) {
return null;
}

DBSPType keyType = new DBSPTypeTuple(keyFields);
DBSPTypeZSet inputType = node.outputType.to(DBSPTypeZSet.class);
return new DBSPTypeIndexedZSet(node.getNode(), keyType, inputType.elementType);
}

public static List<Integer> getKeyFields(DBSPSourceMultisetOperator node) {
List<Integer> keyColumnFields = new ArrayList<>();
int i = 0;
for (InputColumnMetadata inputColumnMetadata : node.metadata.getColumns()) {
if (inputColumnMetadata.isPrimaryKey) {
keyColumnFields.add(i);
}
i++;
}
return keyColumnFields;
}

/**
* Converts {@link DBSPSourceMultisetOperator}s that have a primary key
* into {@link DBSPSourceMapOperator} followed by a {@link DBSPDeindexOperator}.
*/
static class IndexedInputs extends CircuitCloneVisitor {
public IndexedInputs(DBSPCompiler compiler) {
super(compiler, false);
}

@Override
public void postorder(DBSPSourceMultisetOperator node) {
DBSPTypeIndexedZSet ix = getIndexedType(node);
if (ix == null) {
super.postorder(node);
return;
}

List<Integer> keyColumnFields = getKeyFields(node);
DBSPSourceMapOperator set = new DBSPSourceMapOperator(
node.getRelNode(), node.sourceName, keyColumnFields,
ix, node.originalRowType, node.metadata, node.tableName, node.comment);
this.addOperator(set);
DBSPDeindexOperator deindex = new DBSPDeindexOperator(node.getRelNode(), node.getFunctionNode(), set.outputPort());
this.map(node, deindex);
}
}
}
75 changes: 0 additions & 75 deletions ...QL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/IndexedInputs.java
View file
Open in desktop

This file was deleted.

9 changes: 5 additions & 4 deletions ...piler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/OptimizeProjections.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@
import org.dbsp.sqlCompiler.compiler.visitors.inner.ResolveReferences;
import org.dbsp.sqlCompiler.compiler.visitors.inner.Substitution;
import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.FieldUseMap;
import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.FindUnusedFields;
import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.FindUsedFields;
import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.ParameterFieldUse;
import org.dbsp.sqlCompiler.ir.DBSPParameter;
import org.dbsp.sqlCompiler.ir.IDBSPDeclaration;
import org.dbsp.sqlCompiler.ir.IDBSPInnerNode;
Expand Down Expand Up @@ -423,11 +424,11 @@ public void postorder(DBSPApplyOperator operator) {
* required changes, false if they are unchanged. When the operator requires change,
* it is inserted in the circuit and map is remapped to the new operator. */
boolean processAggregate(DBSPStreamAggregateOperator aggregate, DBSPMapIndexOperator map) {
FindUnusedFields unused = new FindUnusedFields(this.compiler);
FindUsedFields unused = new FindUsedFields(this.compiler);
DBSPClosureExpression function = map.getClosureFunction();
Utilities.enforce(function.parameters.length == 1);
unused.findUnusedFields(function);
FieldUseMap useMap = unused.parameterFieldMap.get(function.parameters[0]);
ParameterFieldUse uses = unused.findUsedFields(function);
FieldUseMap useMap = uses.get(function.parameters[0]);
FieldUseMap valueUse = useMap.field(1);
DBSPTypeIndexedZSet ix = aggregate.getOutputIndexedZSetType();

Expand Down
6 changes: 3 additions & 3 deletions ...c/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/monotonicity/InsertLimiters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.dbsp.sqlCompiler.compiler.visitors.monotone.PartiallyMonotoneTuple;
import org.dbsp.sqlCompiler.compiler.visitors.monotone.ScalarMonotoneType;
import org.dbsp.sqlCompiler.compiler.visitors.outer.CircuitCloneVisitor;
import org.dbsp.sqlCompiler.compiler.visitors.outer.IndexedInputs;
import org.dbsp.sqlCompiler.compiler.visitors.outer.ExpandIndexedInputs;
import org.dbsp.sqlCompiler.compiler.visitors.outer.expansion.AggregateDeltaExpansion;
import org.dbsp.sqlCompiler.compiler.visitors.outer.expansion.CommonJoinDeltaExpansion;
import org.dbsp.sqlCompiler.compiler.visitors.outer.expansion.DistinctDeltaExpansion;
Expand Down Expand Up @@ -1591,7 +1591,7 @@ DBSPOperator processLateness(
}
DBSPSourceMultisetOperator multisetInput = operator.as(DBSPSourceMultisetOperator.class);
final DBSPTypeIndexedZSet indexedOutputType = (multisetInput != null) ?
IndexedInputs.getIndexedType(multisetInput) : null;
ExpandIndexedInputs.getIndexedType(multisetInput) : null;

List<DBSPExpression> timestamps = new ArrayList<>();
int index = 0;
Expand Down Expand Up @@ -1649,7 +1649,7 @@ DBSPOperator processLateness(
DBSPInputMapWithWaterlineOperator newSource = null;

if (replaceIndexedInput) {
List<Integer> keyFields = IndexedInputs.getKeyFields(multisetInput);
List<Integer> keyFields = ExpandIndexedInputs.getKeyFields(multisetInput);
// Many of these functions take the key as a parameter, although
// it is a subset of the value fields...
DBSPVariablePath k = indexedOutputType.keyType.ref().var();
Expand Down
41 changes: 41 additions & 0 deletions ...iler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/unusedFields/BorrowedField.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package org.dbsp.sqlCompiler.compiler.visitors.unusedFields;

import java.util.Objects;

/**
* A field of the form &param.i0.i1....in
*/
public final class BorrowedField extends IUsedFields {
private final ParameterField field;

public BorrowedField(ParameterField field) {
this.field = field;
}

@Override
public ParameterFieldUse getParameterUse() {
return this.field.getParameterUse();
}

@Override
public String toString() {
return "&" + this.field;
}

public ParameterField field() {
return field;
}

@Override
public boolean equals(Object obj) {
if (obj == this) return true;
if (obj == null || obj.getClass() != this.getClass()) return false;
var that = (BorrowedField) obj;
return Objects.equals(this.field, that.field);
}

@Override
public int hashCode() {
return Objects.hash(field);
}
}
Loading
Loading