feldera · mihaibudiu · Mar 27, 2026 · Mar 20, 2026
diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java
@@ -26,9 +26,6 @@
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.ParameterException;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.calcite.adapter.jdbc.JdbcSchema;
-import org.apache.calcite.jdbc.CalciteConnection;
-import org.apache.calcite.schema.SchemaPlus;
 import org.apache.calcite.util.Pair;
 import org.dbsp.sqlCompiler.circuit.DBSPCircuit;
 import org.dbsp.sqlCompiler.compiler.CompilerOptions;
@@ -50,7 +47,6 @@
 import org.dbsp.util.Utilities;
 
 import javax.annotation.Nullable;
-import javax.sql.DataSource;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -59,8 +55,6 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.Map;
 
@@ -136,7 +130,7 @@ InputStream getInputFile(@Nullable String inputFile) throws IOException {
     }
 
     /** Run compiler, return exit code. */
-    CompilerMessages run() throws SQLException {
+    CompilerMessages run() {
         DBSPCompiler compiler = new DBSPCompiler(this.options);
         this.options.validate(compiler);
         try {

diff --git a/.../SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWindowOperator.java b/.../SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPWindowOperator.java
@@ -4,14 +4,11 @@
 import org.dbsp.sqlCompiler.circuit.OutputPort;
 import org.dbsp.sqlCompiler.compiler.backend.JsonDecoder;
 import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteEmptyRel;
-import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject;
 import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteRelNode;
 import org.dbsp.sqlCompiler.compiler.visitors.VisitDecision;
 import org.dbsp.sqlCompiler.compiler.visitors.outer.CircuitVisitor;
 import org.dbsp.sqlCompiler.ir.expression.DBSPExpression;
-import org.dbsp.sqlCompiler.ir.expression.NoExpression;
 import org.dbsp.sqlCompiler.ir.type.DBSPType;
-import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeVoid;
 import org.dbsp.util.Utilities;
 
 import javax.annotation.Nullable;

diff --git a/...sp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/ProgramMetadata.java b/...sp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/ProgramMetadata.java
@@ -44,15 +44,14 @@ private static String canonicalVariableName(String variable) {
     }
 
     static final Set<String> reserved = Set.of(
-            DBSPCompiler.WARNINGS_ARE_ERRORS.toLowerCase(Locale.ENGLISH)
+            DBSPCompiler.WARNINGS_ARE_ERRORS.toLowerCase(Locale.ENGLISH),
+            ProgramMetadata.AVOID_STAR_JOINS.toLowerCase(Locale.ENGLISH)
     );
 
     private boolean known(String variable) {
         if (reserved.contains(variable))
             return true;
-        if (variable.startsWith("feldera_ignore_warning_"))
-            return true;
-        return false;
+        return variable.startsWith("feldera_ignore_warning_");
     }
 
     public void set(String variable, DBSPExpression value,

diff --git a/...L-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/dot/ToDotNodesVisitor.java b/...L-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/dot/ToDotNodesVisitor.java
@@ -252,7 +252,7 @@ String getColor(DBSPSimpleOperator operator) {
             case "controlled_filter": return " style=filled fillcolor=cyan";
             case "apply", "apply2", "apply_n": return " style=filled fillcolor=yellow";
             default: break;
-        };
+        }
         if (operator.is(IStateful.class)) {
             // There are more of these every day
             return " style=filled fillcolor=orangered";

diff --git a/...piler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/RustSqlRuntimeLibrary.java b/...piler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/RustSqlRuntimeLibrary.java
@@ -24,7 +24,6 @@
 package org.dbsp.sqlCompiler.compiler.backend.rust;
 
 import org.dbsp.sqlCompiler.compiler.errors.InternalCompilerError;
-import org.dbsp.sqlCompiler.compiler.errors.UnsupportedException;
 import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject;
 import org.dbsp.sqlCompiler.ir.expression.*;
 import org.dbsp.sqlCompiler.ir.type.*;
@@ -35,7 +34,6 @@
 
 import javax.annotation.Nullable;
 import java.util.*;
-import java.util.function.Function;
 
 /** This class encodes (part of) the interface to the SQL
  * runtime library: support functions that implement the SQL semantics. */

diff --git a/...-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java b/...-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java
@@ -3414,7 +3414,7 @@ public DBSPNode compile(RelStatement statement) {
                 // We create an input for the circuit.
                 return this.compileCreateTable(create);
             } else {
-                DropTableStatement drop = statement.as(DropTableStatement.class);
+                DropTableStatement drop = statement.to(DropTableStatement.class);
                 this.compileDropTable(drop);
             }
             return null;

diff --git a/...iler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/aggregates/RangeAggregates.java b/...iler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/aggregates/RangeAggregates.java
@@ -41,18 +41,15 @@
 import org.dbsp.sqlCompiler.ir.expression.literal.DBSPLiteral;
 import org.dbsp.sqlCompiler.ir.type.DBSPType;
 import org.dbsp.sqlCompiler.ir.type.DBSPTypeCode;
-import org.dbsp.sqlCompiler.ir.type.IsDateType;
 import org.dbsp.sqlCompiler.ir.type.IsNumericType;
 import org.dbsp.sqlCompiler.ir.type.IsTimeRelatedType;
 import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeRawTuple;
 import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBaseType;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeDate;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeDecimal;
-import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeDouble;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeLongInterval;
-import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeReal;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeShortInterval;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeTime;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeTimestamp;

diff --git a/...mpiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/aggregates/RankAggregate.java b/...mpiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/aggregates/RankAggregate.java
@@ -25,7 +25,6 @@
 import org.dbsp.sqlCompiler.ir.type.DBSPType;
 import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTupleBase;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger;
-import org.dbsp.util.Utilities;
 
 import java.util.List;
 

diff --git a/...dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlUserDefinedAggregationFunction.java b/...dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlUserDefinedAggregationFunction.java
@@ -9,7 +9,6 @@
 import org.apache.calcite.sql.SqlOperator;
 import org.apache.calcite.sql.type.SqlOperandMetadata;
 import org.apache.calcite.sql.type.SqlOperandTypeChecker;
-import org.apache.calcite.sql.type.SqlOperandTypeInference;
 import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
 import org.apache.calcite.util.Optionality;
 import org.checkerframework.checker.nullness.qual.Nullable;

diff --git a/...main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlCreateFunctionDeclaration.java b/...main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlCreateFunctionDeclaration.java
@@ -16,7 +16,6 @@
 import org.dbsp.util.Utilities;
 
 import javax.annotation.Nullable;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 

diff --git a/...L-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlDropObject.java b/...L-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlDropObject.java
@@ -1,7 +1,6 @@
 package org.dbsp.sqlCompiler.compiler.frontend.parser;
 
 import com.google.common.collect.ImmutableList;
-import org.apache.calcite.jdbc.CalcitePrepare;
 import org.apache.calcite.sql.SqlDrop;
 import org.apache.calcite.sql.SqlIdentifier;
 import org.apache.calcite.sql.SqlNode;

diff --git a/...-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/statements/IHasSchema.java b/...-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/statements/IHasSchema.java
@@ -73,6 +73,7 @@ default void collectPreprocessors(Set<String> result) {
                     return;
 
                 JsonNode json = validation.data();
+                Utilities.enforce(json != null);
                 // None of these "if" statements should "fail" after validation
                 if (json.isArray()) {
                     for (Iterator<JsonNode> it = json.elements(); it.hasNext(); ) {

diff --git a/...SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ChainVisitor.java b/...SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ChainVisitor.java
@@ -20,7 +20,7 @@ public ChainVisitor(DBSPCompiler compiler, CircuitGraphs graphs) {
     }
 
     @Nullable
-    static DBSPChainOperator.Computation getComputation(DBSPSimpleOperator operator) {
+    public static DBSPChainOperator.Computation getComputation(DBSPSimpleOperator operator) {
         if (operator.is(DBSPMapOperator.class)) {
             if (!operator.getFunction().is(DBSPClosureExpression.class))
                 return null;

diff --git a/...compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java b/...compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java
@@ -39,6 +39,8 @@
 import org.dbsp.sqlCompiler.compiler.visitors.inner.CreateRuntimeErrorWrappers;
 import org.dbsp.sqlCompiler.compiler.visitors.inner.Simplify;
 import org.dbsp.sqlCompiler.compiler.visitors.inner.SimplifyWaterline;
+import org.dbsp.sqlCompiler.compiler.visitors.outer.indexSharing.ShareIndexes;
+import org.dbsp.sqlCompiler.compiler.visitors.outer.indexSharing.ShareInputIndexes;
 import org.dbsp.sqlCompiler.compiler.visitors.outer.intern.Intern;
 import org.dbsp.sqlCompiler.compiler.visitors.outer.recursive.RecursiveComponents;
 import org.dbsp.sqlCompiler.compiler.visitors.outer.recursive.ValidateRecursiveOperators;
@@ -133,6 +135,7 @@ void createOptimizer() {
         this.add(new RemoveDeindexOperators(compiler));
         this.add(new OptimizeWithGraph(compiler, g -> new RemoveNoops(compiler, g)));
         this.add(new RemoveIdentityOperators(compiler));
+        this.add(new ShareInputIndexes(compiler));
         this.add(new OptimizeWithGraph(compiler, g -> new ChainVisitor(compiler, g)));
         this.add(new OptimizeWithGraph(compiler,
                 g -> new OptimizeProjections(compiler, false, g, operatorsAnalyzed)));

diff --git a/...compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ExpandAggregates.java b/...compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ExpandAggregates.java
@@ -14,7 +14,6 @@
 import org.dbsp.sqlCompiler.compiler.frontend.TypeCompiler;
 import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteRelNode;
 import org.dbsp.sqlCompiler.ir.aggregate.DBSPAggregateList;
-import org.dbsp.sqlCompiler.ir.aggregate.DBSPFold;
 import org.dbsp.sqlCompiler.ir.aggregate.DBSPMinMax;
 import org.dbsp.sqlCompiler.ir.aggregate.IAggregate;
 import org.dbsp.sqlCompiler.ir.aggregate.LinearAggregate;

diff --git a/...piler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ExpandIndexedInputs.java b/...piler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ExpandIndexedInputs.java
@@ -1,12 +1,22 @@
 package org.dbsp.sqlCompiler.compiler.visitors.outer;
 
 import org.dbsp.sqlCompiler.circuit.operator.DBSPDeindexOperator;
+import org.dbsp.sqlCompiler.circuit.operator.DBSPMapIndexOperator;
 import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceMultisetOperator;
 import org.dbsp.sqlCompiler.circuit.operator.DBSPSourceMapOperator;
 import org.dbsp.sqlCompiler.compiler.DBSPCompiler;
 import org.dbsp.sqlCompiler.compiler.InputColumnMetadata;
+import org.dbsp.sqlCompiler.compiler.visitors.inner.ExpressionTranslator;
+import org.dbsp.sqlCompiler.compiler.visitors.inner.ResolveReferences;
 import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.FindCommonProjections;
 import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.ReplaceCommonProjections;
+import org.dbsp.sqlCompiler.ir.DBSPParameter;
+import org.dbsp.sqlCompiler.ir.IDBSPInnerNode;
+import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression;
+import org.dbsp.sqlCompiler.ir.expression.DBSPDerefExpression;
+import org.dbsp.sqlCompiler.ir.expression.DBSPExpression;
+import org.dbsp.sqlCompiler.ir.expression.DBSPFieldExpression;
+import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath;
 import org.dbsp.sqlCompiler.ir.type.DBSPType;
 import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeIndexedZSet;
 import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple;
@@ -26,6 +36,7 @@ public ExpandIndexedInputs(DBSPCompiler compiler) {
         FindCommonProjections fcp = new FindCommonProjections(compiler, graph.getGraphs());
         this.add(fcp);
         this.add(new ReplaceCommonProjections(compiler, fcp));
+        this.add(new DeadCode(compiler, true));
     }
 
     /**
@@ -90,5 +101,111 @@ public void postorder(DBSPSourceMultisetOperator node) {
             DBSPDeindexOperator deindex = new DBSPDeindexOperator(node.getRelNode(), node.getFunctionNode(), set.outputPort());
             this.map(node, deindex);
         }
+
+
+        @Override
+        public void postorder(DBSPMapIndexOperator node) {
+            // Note: map is the ORIGINAL input of this node
+            var multiset = node.input().node().as(DBSPSourceMultisetOperator.class);
+            if (multiset == null) {
+                super.postorder(node);
+                return;
+            }
+
+            List<Integer> keyColumnFields = getKeyFields(multiset);
+            if (keyColumnFields.isEmpty()) {
+                super.postorder(node);
+                return;
+            }
+
+            // Find the translation, it must already exist
+            var deindex = this.mapped(multiset.outputPort());
+            DBSPSourceMapOperator map = deindex.node().inputs.get(0).node().to(DBSPSourceMapOperator.class);
+            DBSPClosureExpression rewritten = this.rewriteClosure(node.getClosureFunction(), map, keyColumnFields);
+            var mx = new DBSPMapIndexOperator(node.getRelNode(), rewritten, map.outputPort());
+            this.map(node, mx);
+        }
+
+        private DBSPClosureExpression rewriteClosure(
+                DBSPClosureExpression closure, DBSPSourceMapOperator map, List<Integer> keyColumnFields) {
+            DBSPVariablePath w = map.getOutputIndexedZSetType().getKVRefType().var();
+            IndexFunctionRewriter rewriter = new IndexFunctionRewriter(
+                    this.compiler, closure.parameters[0], w, keyColumnFields);
+            var result = rewriter.apply(closure);
+            return result.to(DBSPClosureExpression.class);
+        }
+    }
+
+    static class IndexFunctionRewriter extends ExpressionTranslator {
+        // closure is a closure of the form |v: &TupN<>| -> (TupK<>, TupM<>)
+        // map is an operator with an output of type (TupR<>, TupN<>), N >= R
+        // keyColumnFields is the list of fields of map which are keys.
+        // There are R such fields, and TupR<> is composed of these fields of the TupL<> tuple
+        // This rewrites the closure to have the form |w: (&TupR<>, &TupN<>)| -> (TupK<>, TupM<>)
+        // and to use the key fields as much as possible.  So
+        // (*v.X) is rewritten as *(w.0).Y if Y is the X-th key field or as
+        // *(w.1).X otherwise
+        final ResolveReferences resolver;
+        private final DBSPParameter parameter;
+        private final DBSPVariablePath w;
+        private final List<Integer> keyColumnFields;
+
+        public IndexFunctionRewriter(
+                DBSPCompiler compiler, DBSPParameter parameter,
+                DBSPVariablePath w, List<Integer> keyColumnFields) {
+            super(compiler);
+            this.resolver = new ResolveReferences(compiler, false);
+            this.parameter = parameter;
+            this.w = w;
+            this.keyColumnFields = keyColumnFields;
+        }
+
+        @Override
+        public void startVisit(IDBSPInnerNode node) {
+            super.startVisit(node);
+            this.resolver.apply(node);
+        }
+
+        @Override
+        public void postorder(DBSPVariablePath var) {
+            var decl = this.resolver.reference.getDeclaration(var);
+            if (decl == this.parameter) {
+                this.map(var, this.w.field(1));
+            } else {
+                super.postorder(var);
+            }
+        }
+
+        @Override
+        public void postorder(DBSPFieldExpression field) {
+            if (field.expression.is(DBSPDerefExpression.class)) {
+                var deref = field.expression.to(DBSPDerefExpression.class);
+                if (deref.expression.is(DBSPVariablePath.class)) {
+                    var var = deref.expression.to(DBSPVariablePath.class);
+                    var decl = this.resolver.reference.getDeclaration(var);
+                    if (decl == this.parameter) {
+                        // field is of the form (*v.X)
+                        int keyField = this.keyColumnFields.indexOf(field.fieldNo);
+                        if (keyField < 0) {
+                            this.map(field, w.field(1).deref().field(field.fieldNo));
+                        } else {
+                            this.map(field, w.field(0).deref().field(keyField));
+                        }
+                        return;
+                    }
+                }
+            }
+            super.postorder(field);
+        }
+
+        @Override
+        public void postorder(DBSPClosureExpression closure) {
+            if (this.context.isEmpty()) {
+                DBSPExpression body = this.getE(closure.body);
+                this.map(closure, body.closure(this.w.asParameter()));
+            } else {
+                super.postorder(closure);
+            }
+        }
     }
 }