feldera · mihaibudiu · Jun 18, 2024 · Jun 18, 2024
diff --git a/docs/sql/table.md b/docs/sql/table.md
@@ -71,7 +71,9 @@ HOP(data, DESCRIPTOR(timecol), slide, size [, offset ])
 
 Indicates a hopping window for `timecol`, covering rows within the
 interval of `size`, shifting every `slide` and optionally aligned at
-`offset`.  The type of the `timecol` has to be `TIMESTAMP`.
+`offset`.  The type of the `timecol` has to be `TIMESTAMP`.  The
+intervals must be compile-time constants, and be expressed as a
+"short" interval (i.e., days or smaller time units).
 
 Here is an example:
 

diff --git a/...ler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPHopOperator.java b/...ler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/circuit/operator/DBSPHopOperator.java
@@ -0,0 +1,58 @@
+package org.dbsp.sqlCompiler.circuit.operator;
+
+import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject;
+import org.dbsp.sqlCompiler.compiler.visitors.VisitDecision;
+import org.dbsp.sqlCompiler.compiler.visitors.outer.CircuitVisitor;
+import org.dbsp.sqlCompiler.ir.expression.DBSPExpression;
+import org.dbsp.sqlCompiler.ir.type.DBSPType;
+import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeZSet;
+
+import javax.annotation.Nullable;
+import java.util.List;
+
+/** Implements the table function HOP.  This is desugared into a pair of
+ * operators: map (computing the hop start window) followed by flat_map
+ * (which generates all the windows).  It does not correspond to any DBSP
+ * Rust operator. */
+public final class DBSPHopOperator extends DBSPUnaryOperator {
+    public final int timestampIndex;
+    public final DBSPExpression interval;
+    public final DBSPExpression start;
+    public final DBSPExpression size;
+
+    public DBSPHopOperator(CalciteObject node, int timestampIndex,
+                           DBSPExpression interval,
+                           DBSPExpression start, DBSPExpression size,
+                           DBSPTypeZSet outputType, DBSPOperator input) {
+        super(node, "hop", null, outputType, input.isMultiset, input);
+        this.timestampIndex = timestampIndex;
+        this.interval = interval;
+        this.start = start;
+        this.size = size;
+    }
+
+    @Override
+    public void accept(CircuitVisitor visitor) {
+        visitor.push(this);
+        VisitDecision decision = visitor.preorder(this);
+        if (!decision.stop())
+            visitor.postorder(this);
+        visitor.pop(this);
+    }
+
+    @Override
+    public DBSPOperator withFunction(@Nullable DBSPExpression expression, DBSPType outputType) {
+        return new DBSPHopOperator(
+                this.getNode(), this.timestampIndex, this.interval, this.start, this.size,
+                outputType.to(DBSPTypeZSet.class), this.input());
+    }
+
+    @Override
+    public DBSPOperator withInputs(List<DBSPOperator> newInputs, boolean force) {
+        if (force || this.inputsDiffer(newInputs))
+            return new DBSPHopOperator(
+                    this.getNode(), this.timestampIndex, this.interval, this.start, this.size,
+                    this.getOutputZSetType(), newInputs.get(0));
+        return this;
+    }
+}
diff --git a/...mpiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/ToDotVisitor.java b/...mpiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/ToDotVisitor.java
@@ -181,6 +181,14 @@ String getColor(DBSPOperator operator) {
             case "controlled_filter" -> " style=filled fillcolor=cyan";
             case "apply" -> " style=filled fillcolor=yellow";
             case "integrate_trace_retain_keys", "integrate_trace_retain_values" -> " style=filled fillcolor=pink";
+            // stateful operators
+            case "distinct", "stream_distinct",
+                 "aggregate", "partitioned_rolling_aggregate", "aggregate_linear",
+                 "stream_aggregate", "stream_aggregate_linear", "partitioned_rolling_aggregate_with_waterline",
+                 "partitioned_tree_aggregate",
+                 "join", "stream_join", "join_flatmap",
+                 "delay_trace", "delay", "differentiate", "topK", "integrate",
+                 "lag_custom_order", "upsert" -> " style=filled fillcolor=red";
             default -> "";
         };
     }

diff --git a/...SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/errors/UnsupportedException.java b/...SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/errors/UnsupportedException.java
@@ -40,11 +40,11 @@ public UnsupportedException(CalciteObject obj) {
     }
 
     public UnsupportedException(String msg, CalciteObject obj) {
-        super(msg, obj);
+        super(msg + ": " + obj, obj);
     }
 
     public UnsupportedException(String msg, IHasCalciteObject obj) {
-        super(msg, obj.getNode());
+        this(msg, obj.getNode());
     }
 
     @Override

diff --git a/...-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java b/...-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java
@@ -77,6 +77,7 @@
 import org.dbsp.sqlCompiler.circuit.operator.DBSPDifferentiateOperator;
 import org.dbsp.sqlCompiler.circuit.operator.DBSPFilterOperator;
 import org.dbsp.sqlCompiler.circuit.operator.DBSPFlatMapOperator;
+import org.dbsp.sqlCompiler.circuit.operator.DBSPHopOperator;
 import org.dbsp.sqlCompiler.circuit.operator.DBSPIndexedTopKOperator;
 import org.dbsp.sqlCompiler.circuit.operator.DBSPIntegrateOperator;
 import org.dbsp.sqlCompiler.circuit.operator.DBSPLagOperator;
@@ -122,7 +123,6 @@
 import org.dbsp.sqlCompiler.ir.expression.DBSPApplyExpression;
 import org.dbsp.sqlCompiler.ir.expression.DBSPApplyMethodExpression;
 import org.dbsp.sqlCompiler.ir.expression.DBSPBinaryExpression;
-import org.dbsp.sqlCompiler.ir.expression.DBSPBlockExpression;
 import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression;
 import org.dbsp.sqlCompiler.ir.expression.DBSPComparatorExpression;
 import org.dbsp.sqlCompiler.ir.expression.DBSPConstructorExpression;
@@ -142,14 +142,13 @@
 import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath;
 import org.dbsp.sqlCompiler.ir.expression.literal.DBSPBoolLiteral;
 import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI32Literal;
+import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntervalMillisLiteral;
 import org.dbsp.sqlCompiler.ir.expression.literal.DBSPLiteral;
 import org.dbsp.sqlCompiler.ir.expression.literal.DBSPZSetLiteral;
 import org.dbsp.sqlCompiler.ir.path.DBSPPath;
 import org.dbsp.sqlCompiler.ir.path.DBSPSimplePathSegment;
 import org.dbsp.sqlCompiler.ir.statement.DBSPFunctionItem;
 import org.dbsp.sqlCompiler.ir.statement.DBSPItem;
-import org.dbsp.sqlCompiler.ir.statement.DBSPLetStatement;
-import org.dbsp.sqlCompiler.ir.statement.DBSPStatement;
 import org.dbsp.sqlCompiler.ir.statement.DBSPStructItem;
 import org.dbsp.sqlCompiler.ir.statement.DBSPStructWithHelperItem;
 import org.dbsp.sqlCompiler.ir.type.DBSPType;
@@ -163,6 +162,7 @@
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBool;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeDate;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger;
+import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeMillisInterval;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeTimestamp;
 import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeVoid;
 import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeIndexedZSet;
@@ -240,7 +240,7 @@ private DBSPType convertType(RelDataType dt, boolean asStruct) {
         return this.compiler.getTypeCompiler().convertType(dt, asStruct);
     }
 
-    private DBSPTypeZSet makeZSet(DBSPType type) {
+    DBSPTypeZSet makeZSet(DBSPType type) {
         return TypeCompiler.makeZSet(type);
     }
 
@@ -455,67 +455,26 @@ void compileHop(LogicalTableFunctionScan scan, RexCall call) {
         List<RexNode> operands = call.getOperands();
         assert call.operandCount() == 3 || call.operandCount() == 4;
         int timestampIndex = this.getDescriptor(operands.get(0));
+        // Need to do a few more checks, Calcite should be doing these really.
         DBSPExpression interval = expressionCompiler.compile(operands.get(1));
-        DBSPExpression start = null;
+        if (!interval.getType().is(DBSPTypeMillisInterval.class)) {
+            throw new UnsupportedException("Hopping window intervals must be 'short' SQL intervals (days and lower)",
+                    interval.getNode());
+        }
+        DBSPExpression start;
         DBSPExpression size = expressionCompiler.compile(operands.get(2));
+        if (!size.getType().is(DBSPTypeMillisInterval.class)) {
+            throw new UnsupportedException("Hopping window intervals must be 'short' SQL intervals (days and lower)",
+                    size.getNode());
+        }
         if (call.operandCount() == 4)
             start = expressionCompiler.compile(operands.get(3));
+        else
+            start = new DBSPIntervalMillisLiteral(0, false);
 
-        DBSPExpression[] results = new DBSPExpression[inputRowType.size() + 1];
-        for (int i = 0; i < inputRowType.size(); i++) {
-            results[i] = row.deref().field(i).applyCloneIfNeeded();
-        }
-        int nextIndex = inputRowType.size();
-
-        // Map builds the array
-        List<DBSPExpression> hopArguments = new ArrayList<>();
-        hopArguments.add(row.deref().field(timestampIndex));
-        hopArguments.add(interval);
-        hopArguments.add(size);
-        if (start != null)
-            hopArguments.add(start);
-        DBSPType hopType = type.tupFields[nextIndex];
-        results[nextIndex] = ExpressionCompiler.compilePolymorphicFunction(
-                "hop", node, new DBSPTypeVec(hopType, false), hopArguments, 3, 4);
-        DBSPTupleExpression mapBody = new DBSPTupleExpression(results);
-        DBSPClosureExpression func = mapBody.closure(row.asParameter());
-        DBSPMapOperator map = new DBSPMapOperator(node, func, makeZSet(mapBody.getType()), opInput);
-        this.circuit.addOperator(map);
-
-        // Flatmap flattens the array
-        DBSPVariablePath data = new DBSPVariablePath("data", mapBody.getType().ref());
-        // This is not the timestamp type, since e can never be null.
-        DBSPVariablePath e = new DBSPVariablePath("e", hopType);
-        DBSPExpression collectionExpression = data.deref().field(nextIndex).borrow();
-
-        List<DBSPStatement> statements = new ArrayList<>();
-        String varName = "x";
-        for (int i = 0; i < inputRowType.size(); i++) {
-            DBSPLetStatement stat = new DBSPLetStatement(varName + i, data.deref().field(i).applyCloneIfNeeded());
-            statements.add(stat);
-        }
-        DBSPLetStatement array = new DBSPLetStatement("array", collectionExpression);
-        statements.add(array);
-        DBSPLetStatement clone = new DBSPLetStatement("array_clone", array.getVarReference().deref().applyClone());
-        statements.add(clone);
-        DBSPExpression iter = new DBSPApplyMethodExpression("into_iter",
-                DBSPTypeAny.getDefault(), clone.getVarReference().applyClone());
-        DBSPExpression[] resultFields = new DBSPExpression[type.size()];
-        for (int i = 0; i < inputRowType.size(); i++)
-            resultFields[i] = statements.get(i).to(DBSPLetStatement.class).getVarReference().applyClone();
-        nextIndex = inputRowType.size();
-        resultFields[nextIndex++] = e;
-        resultFields[nextIndex] = ExpressionCompiler.makeBinaryExpression(node,
-                // not the timestampType, but the hopType
-                hopType, DBSPOpcode.ADD, e, size);
-
-        DBSPExpression toTuple = new DBSPTupleExpression(resultFields).closure(e.asParameter());
-        DBSPExpression makeTuple = new DBSPApplyMethodExpression(node,
-                "map", DBSPTypeAny.getDefault(), iter, toTuple);
-        DBSPBlockExpression block = new DBSPBlockExpression(statements, makeTuple);
-
-        DBSPOperator result = new DBSPFlatMapOperator(node, block.closure(data.asParameter()), makeZSet(type), map);
-        this.assignOperator(scan, result);
+        DBSPHopOperator hop = new DBSPHopOperator(
+                node, timestampIndex, interval, start, size, makeZSet(type), opInput);
+        this.assignOperator(scan, hop);
     }
 
     void visitTableFunction(LogicalTableFunctionScan scan) {

diff --git a/...SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java b/...SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java
@@ -440,8 +440,8 @@ static String getCallName(RexCall call) {
         return call.op.getName().toLowerCase();
     }
 
-    static DBSPExpression compilePolymorphicFunction(String opName, CalciteObject node, DBSPType resultType,
-                                              List<DBSPExpression> ops, Integer... expectedArgCount) {
+    public static DBSPExpression compilePolymorphicFunction(String opName, CalciteObject node, DBSPType resultType,
+                                                            List<DBSPExpression> ops, Integer... expectedArgCount) {
         validateArgCount(node, ops.size(), expectedArgCount);
         StringBuilder functionName = new StringBuilder(opName);
         DBSPExpression[] operands = ops.toArray(new DBSPExpression[0]);

diff --git a/...java/org/dbsp/sqlCompiler/compiler/visitors/inner/monotone/MonotoneTransferFunctions.java b/...java/org/dbsp/sqlCompiler/compiler/visitors/inner/monotone/MonotoneTransferFunctions.java
@@ -281,8 +281,11 @@ public void postorder(DBSPBinaryExpression expression) {
                         left.getReducedExpression(), right.getReducedExpression());
             }
         }
-        if (left.mayBeMonotone() && expression.operation == DBSPOpcode.DIV) {
-            // Dividing a monotone expression by a positive constant produces a monotone result
+        if (left.mayBeMonotone() &&
+                (expression.operation == DBSPOpcode.DIV || expression.operation == DBSPOpcode.MUL)) {
+            // Multiplying or dividing a monotone expression by
+            // a positive constant produces a monotone result
+            // TODO: multiplication is commutative.
             if (expression.right.is(DBSPLiteral.class)) {
                 if (expression.right.is(IsNumericLiteral.class)) {
                     if (expression.right.to(IsNumericLiteral.class).gt0()) {
@@ -373,7 +376,8 @@ public void postorder(DBSPApplyExpression expression) {
                         name.startsWith("numeric_inc") ||
                         name.startsWith("extract_year_") ||
                         name.startsWith("extract_epoch_") ||
-                        name.startsWith("extract_hour_Time")
+                        name.startsWith("extract_hour_Time") ||
+                        name.equals("hop_start_timestamp")
                 ) {
                     resultType = new MonotoneType(expression.getType());
                     reduced = expression.replaceArguments(reducedArgs);

diff --git a/...piler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitCloneVisitor.java b/...piler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitCloneVisitor.java
@@ -239,6 +239,11 @@ public void postorder(DBSPFlatMapOperator operator) {
         this.replace(operator);
     }
 
+    @Override
+    public void postorder(DBSPHopOperator operator) {
+        this.replace(operator);
+    }
+
     @Override
     public void postorder(DBSPIntegrateOperator operator) {
         this.replace(operator);

diff --git a/...compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java b/...compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java
@@ -78,6 +78,7 @@ CircuitTransform getOptimizer() {
             if (options.languageOptions.incrementalize)
                 passes.add(new NoIntegralVisitor(reporter));
         }
+        passes.add(new ExpandHop(reporter));
         passes.add(new RemoveDeindexOperators(reporter));
         passes.add(new RemoveViewOperators(reporter));
         passes.add(new EliminateFunctions(reporter).circuitRewriter());

diff --git a/...L-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitVisitor.java b/...L-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitVisitor.java
@@ -220,6 +220,10 @@ public VisitDecision preorder(DBSPFlatMapOperator node) {
         return this.preorder(node.to(DBSPUnaryOperator.class));
     }
 
+    public VisitDecision preorder(DBSPHopOperator node) {
+        return this.preorder(node.to(DBSPUnaryOperator.class));
+    }
+
     public VisitDecision preorder(DBSPFilterOperator node) {
         return this.preorder(node.to(DBSPUnaryOperator.class));
     }
@@ -430,6 +434,10 @@ public void postorder(DBSPFlatMapOperator node) {
         this.postorder(node.to(DBSPUnaryOperator.class));
     }
 
+    public void postorder(DBSPHopOperator node) {
+        this.postorder(node.to(DBSPUnaryOperator.class));
+    }
+
     public void postorder(DBSPFilterOperator node) {
         this.postorder(node.to(DBSPUnaryOperator.class));
     }