Skip to content

[SQL] Support hopping windows for nullable timestamp columns #1887

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mihaibudiu merged 1 commit into from
Jun 18, 2024
Merged

[SQL] Support hopping windows for nullable timestamp columns #1887

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions docs/sql/table.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ HOP(data, DESCRIPTOR(timecol), slide, size [, offset ])

Indicates a hopping window for `timecol`, covering rows within the
interval of `size`, shifting every `slide` and optionally aligned at
`offset`.
`offset`. The type of the `timecol` has to be `TIMESTAMP`.

Here is an example:

Expand All @@ -93,6 +93,7 @@ SELECT * FROM TABLE(
SIZE => INTERVAL '5' MINUTE));
```

applies hopping with 5-minute interval size on rows from table orders
and shifting every 2 minutes. rowtime is the column of table orders
that tells data completeness.
applies hopping with 5-minute interval size on rows from table
`orders` and shifting every 2 minutes.

A `NULL` timestamp produces no rows in the result.
31 changes: 12 additions & 19 deletions ...-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,14 +353,12 @@ void visitCorrelate(LogicalCorrelate correlate) {
rightProject = (Project) correlateRight;
correlateRight = rightProject.getInput(0);
}
if (!(correlateRight instanceof Uncollect))
if (!(correlateRight instanceof Uncollect uncollect))
throw new UnimplementedException(node);
Uncollect uncollect = (Uncollect) correlateRight;
CalciteObject uncollectNode = CalciteObject.create(uncollect);
RelNode uncollectInput = uncollect.getInput();
if (!(uncollectInput instanceof LogicalProject))
if (!(uncollectInput instanceof LogicalProject project))
throw new UnimplementedException(node);
LogicalProject project = (LogicalProject) uncollectInput;
if (project.getProjects().size() != 1)
throw new UnimplementedException(node);
RexNode projection = project.getProjects().get(0);
Expand Down Expand Up @@ -471,7 +469,6 @@ void compileHop(LogicalTableFunctionScan scan, RexCall call) {

// Map builds the array
List<DBSPExpression> hopArguments = new ArrayList<>();
DBSPType timestampType = row.deref().field(timestampIndex).getType();
hopArguments.add(row.deref().field(timestampIndex));
hopArguments.add(interval);
hopArguments.add(size);
Expand All @@ -487,7 +484,8 @@ void compileHop(LogicalTableFunctionScan scan, RexCall call) {

// Flatmap flattens the array
DBSPVariablePath data = new DBSPVariablePath("data", mapBody.getType().ref());
DBSPVariablePath e = new DBSPVariablePath("e", timestampType);
// This is not the timestamp type, since e can never be null.
DBSPVariablePath e = new DBSPVariablePath("e", hopType);
DBSPExpression collectionExpression = data.deref().field(nextIndex).borrow();

List<DBSPStatement> statements = new ArrayList<>();
Expand All @@ -508,7 +506,8 @@ void compileHop(LogicalTableFunctionScan scan, RexCall call) {
nextIndex = inputRowType.size();
resultFields[nextIndex++] = e;
resultFields[nextIndex] = ExpressionCompiler.makeBinaryExpression(node,
timestampType, DBSPOpcode.ADD, e, size);
// not the timestampType, but the hopType
hopType, DBSPOpcode.ADD, e, size);

DBSPExpression toTuple = new DBSPTupleExpression(resultFields).closure(e.asParameter());
DBSPExpression makeTuple = new DBSPApplyMethodExpression(node,
Expand Down Expand Up @@ -1356,13 +1355,11 @@ void generateNestedTopK(LogicalWindow window, Window.Group group, int limit, Sql
* @return -1 if the operands don't have the expected shape. The limit value otherwise.
*/
int limitValue(RexNode compared, int expectedIndex, RexNode limit, boolean eq) {
if (compared instanceof RexInputRef) {
RexInputRef ri = (RexInputRef) compared;
if (compared instanceof RexInputRef ri) {
if (ri.getIndex() != expectedIndex)
return -1;
}
if (limit instanceof RexLiteral) {
RexLiteral literal = (RexLiteral) limit;
if (limit instanceof RexLiteral literal) {
SqlTypeName type = literal.getType().getSqlTypeName();
if (SqlTypeName.INT_TYPES.contains(type)) {
Integer value = literal.getValueAs(Integer.class);
Expand Down Expand Up @@ -1839,10 +1836,8 @@ void visitWindow(LogicalWindow window) {
// Aggregate functions seem always to be at the end.
// The window always seems to collect all fields.
int aggregationArgumentIndex = inputRowType.size();
if (operator instanceof SqlRankFunction) {
SqlRankFunction rank = (SqlRankFunction) operator;
if (parent instanceof LogicalFilter) {
LogicalFilter filter = (LogicalFilter) parent;
if (operator instanceof SqlRankFunction rank) {
if (parent instanceof LogicalFilter filter) {
RexNode condition = filter.getCondition();
if (condition instanceof RexCall) {
int limit = this.isLimit((RexCall) condition, aggregationArgumentIndex);
Expand Down Expand Up @@ -2149,9 +2144,8 @@ DBSPNode compileModifyTable(TableModifyStatement modify) {
this.modifyTableTranslation = new ModifyTableTranslation(
modify, def, targetColumnList, this.compiler);
DBSPZSetLiteral result;
if (modify.rel instanceof LogicalTableScan) {
if (modify.rel instanceof LogicalTableScan scan) {
// Support for INSERT INTO table (SELECT * FROM otherTable)
LogicalTableScan scan = (LogicalTableScan) modify.rel;
List<String> name = scan.getTable().getQualifiedName();
String sourceTable = name.get(name.size() - 1);
result = this.tableContents.getTableContents(sourceTable);
Expand Down Expand Up @@ -2220,8 +2214,7 @@ DBSPNode compileCreateTable(CreateTableStatement create) {
DBSPType rowType = def.getRowTypeAsTuple(this.compiler.getTypeCompiler());
DBSPTypeStruct originalRowType = def.getRowTypeAsStruct(this.compiler.getTypeCompiler());
CalciteObject identifier = CalciteObject.EMPTY;
if (create.node instanceof SqlCreateTable) {
SqlCreateTable sct = (SqlCreateTable)create.node;
if (create.node instanceof SqlCreateTable sct) {
identifier = CalciteObject.create(sct.name);
}
List<InputColumnMetadata> metadata = Linq.map(create.columns, this::convertMetadata);
Expand Down
10 changes: 5 additions & 5 deletions ...dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/SqlIoTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,17 +200,17 @@ public void q(String queryAndOutput) {
* @param query Query to run.
* @param messageFragment This fragment should appear in the error message. */
public void queryFailingInCompilation(String query, String messageFragment) {
this.statementFailingInCompilation("CREATE VIEW VV AS " + query, messageFragment);
this.statementsFailingInCompilation("CREATE VIEW VV AS " + query, messageFragment);
}

/** Run a statement that is expected to fail in compilation.
* @param statement Statement to compile.
/** Run one or more statements that are expected to fail in compilation.
* @param statements Statements to compile.
* @param messageFragment This fragment should appear in the error message. */
public void statementFailingInCompilation(String statement, String messageFragment) {
public void statementsFailingInCompilation(String statements, String messageFragment) {
DBSPCompiler compiler = this.testCompiler();
compiler.options.languageOptions.throwOnError = false;
this.prepareInputs(compiler);
compiler.compileStatement(statement);
compiler.compileStatements(statements);
compiler.optimize();
Assert.assertTrue(compiler.messages.exitCode != 0);
String message = compiler.messages.toString();
Expand Down
2 changes: 1 addition & 1 deletion ...SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/functions/FunctionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ CREATE FUNCTION contains_number(str VARCHAR NOT NULL, value INTEGER)

@Test
public void testTypeError() {
this.statementFailingInCompilation("CREATE FUNCTION error(x INTEGER) RETURNS INTEGER AS ''",
this.statementsFailingInCompilation("CREATE FUNCTION error(x INTEGER) RETURNS INTEGER AS ''",
"should return");
}

Expand Down
59 changes: 59 additions & 0 deletions ...QL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/streaming/StreamingTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,65 @@ CREATE TABLE series (
this.addRustTestCase("hoppingTest", ccs);
}

@Test
public void nullableHoppingTest() {
String sql = """
CREATE TABLE series (
pickup TIMESTAMP
);
CREATE VIEW V AS
SELECT * FROM TABLE(
HOP(
TABLE series,
DESCRIPTOR(pickup),
INTERVAL '2' MINUTE,
INTERVAL '5' MINUTE));""";
DBSPCompiler compiler = this.testCompiler();
compiler.compileStatements(sql);
CompilerCircuitStream ccs = new CompilerCircuitStream(compiler);
this.addRustTestCase("nullableHoppingTest", ccs);
}

@Test
public void variableHoppingTest() {
String sql = """
CREATE TABLE series (
pickup TIMESTAMP
);
CREATE VIEW V AS
SELECT * FROM TABLE(
HOP(
TABLE series,
DESCRIPTOR(pickup),
NULL,
NULL));""";
this.statementsFailingInCompilation(sql, "Cannot apply 'HOP'");
sql = """
CREATE TABLE series (
pickup TIMESTAMP
);
CREATE VIEW V AS
SELECT * FROM TABLE(
HOP(
TABLE series,
DESCRIPTOR(pickup),
6,
DATE '2020-12-20'));""";
this.statementsFailingInCompilation(sql, "Cannot apply 'HOP'");
sql = """
CREATE TABLE series (
pickup TIMESTAMP
);
CREATE VIEW V AS
SELECT * FROM TABLE(
HOP(
TABLE series,
DESCRIPTOR(pickup),
DESCRIPTOR(column),
INTERVAL 1 HOUR));""";
this.statementsFailingInCompilation(sql, "Cannot apply 'HOP'");
}

@Test
public void tumblingTestLimits() {
String sql = """
Expand Down
20 changes: 10 additions & 10 deletions sql-to-dbsp-compiler/lib/sqllib/src/timestamp.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -596,16 +596,16 @@ pub fn hop_Timestamp_ShortInterval_ShortInterval(
result
}

some_polymorphic_function3!(
hop,
Timestamp,
Timestamp,
ShortInterval,
ShortInterval,
ShortInterval,
ShortInterval,
Vec<Timestamp>
);
pub fn hop_TimestampN_ShortInterval_ShortInterval(
ts: Option<Timestamp>,
period: ShortInterval,
size: ShortInterval,
) -> Vec<Timestamp> {
match ts {
None => Vec::new(),
Some(ts) => hop_Timestamp_ShortInterval_ShortInterval(ts, period, size),
}
}

//////////////////////////// Date

Expand Down