Skip to content

Commit fe161ea

Browse files
committed
Fix natural-month subtraction in date expressions
A date expression such as `2024-03-15 - 1mo` computed the length of the month after the given time and subtracted it, giving 2024-02-13 instead of 2024-02-15 (and `2024-03-31 - 1mo` gave 2024-03-01). Apply the subtracted duration backwards from the original time, so that `- Nmo` goes back N calendar months the same way `+ Nmo` goes forward. This affects both the tree and table date-expression parsers. Addition, other units, and the context-free duration path are unchanged. Fixes #19210
1 parent a4568b6 commit fe161ea

4 files changed

Lines changed: 113 additions & 20 deletions

File tree

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3677,13 +3677,15 @@ public Long parseDateExpression(IoTDBSqlParser.DateExpressionContext ctx, String
36773677
time = parseDateTimeFormat(ctx.getChild(0).getText());
36783678
for (int i = 1; i < ctx.getChildCount(); i = i + 2) {
36793679
try {
3680-
long duration =
3680+
// apply "- duration" backwards from time, so that natural months are counted correctly
3681+
long offset =
36813682
DataNodeDateTimeUtils.convertDurationStrToLong(
3682-
time, ctx.getChild(i + 1).getText(), precision, false);
3683-
time =
3684-
"+".equals(ctx.getChild(i).getText())
3685-
? Math.addExact(time, duration)
3686-
: Math.subtractExact(time, duration);
3683+
time,
3684+
ctx.getChild(i + 1).getText(),
3685+
precision,
3686+
false,
3687+
"-".equals(ctx.getChild(i).getText()));
3688+
time = Math.addExact(time, offset);
36873689
} catch (ArithmeticException e) {
36883690
throw new SemanticException(
36893691
String.format(
@@ -3699,13 +3701,11 @@ private Long parseDateExpression(IoTDBSqlParser.DateExpressionContext ctx, long
36993701
time = parseDateTimeFormat(ctx.getChild(0).getText(), currentTime, zoneId);
37003702
for (int i = 1; i < ctx.getChildCount(); i = i + 2) {
37013703
try {
3702-
long duration =
3704+
// apply "- duration" backwards from time, so that natural months are counted correctly
3705+
long offset =
37033706
DataNodeDateTimeUtils.convertDurationStrToLong(
3704-
time, ctx.getChild(i + 1).getText(), false);
3705-
time =
3706-
"+".equals(ctx.getChild(i).getText())
3707-
? Math.addExact(time, duration)
3708-
: Math.subtractExact(time, duration);
3707+
time, ctx.getChild(i + 1).getText(), false, "-".equals(ctx.getChild(i).getText()));
3708+
time = Math.addExact(time, offset);
37093709
} catch (ArithmeticException e) {
37103710
throw new SemanticException(
37113711
String.format(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3566,13 +3566,11 @@ private Long parseDateExpression(
35663566
time = parseDateTimeFormat(ctx.getChild(0).getText(), currentTime, zoneId);
35673567
for (int i = 1; i < ctx.getChildCount(); i = i + 2) {
35683568
try {
3569-
long duration =
3569+
// apply "- duration" backwards from time, so that natural months are counted correctly
3570+
long offset =
35703571
DataNodeDateTimeUtils.convertDurationStrToLong(
3571-
time, ctx.getChild(i + 1).getText(), false);
3572-
time =
3573-
"+".equals(ctx.getChild(i).getText())
3574-
? Math.addExact(time, duration)
3575-
: Math.subtractExact(time, duration);
3572+
time, ctx.getChild(i + 1).getText(), false, "-".equals(ctx.getChild(i).getText()));
3573+
time = Math.addExact(time, offset);
35763574
} catch (ArithmeticException e) {
35773575
throw new SemanticException(
35783576
String.format(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,22 @@ public static long convertDurationStrToLongForTest(
164164
*/
165165
public static long convertDurationStrToLong(
166166
long currentTime, String duration, String timestampPrecision, boolean convertYearToMonth) {
167+
return convertDurationStrToLong(
168+
currentTime, duration, timestampPrecision, convertYearToMonth, false);
169+
}
170+
171+
/**
172+
* Same as {@link #convertDurationStrToLong(long, String, String, boolean)}, but if {@code
173+
* negative} is true, every unit is applied backwards from {@code currentTime} and the returned
174+
* offset is negative. This is what {@code currentTime - duration} means for natural months, e.g.
175+
* 2024-03-15 and 1mo gives -29 days (to 2024-02-15), not -31 days (the length of the next month).
176+
*/
177+
public static long convertDurationStrToLong(
178+
long currentTime,
179+
String duration,
180+
String timestampPrecision,
181+
boolean convertYearToMonth,
182+
boolean negative) {
167183
try {
168184
long total = 0;
169185
long temp = 0;
@@ -188,7 +204,7 @@ public static long convertDurationStrToLong(
188204
total,
189205
convertDurationStrToLong(
190206
currentTime == -1 ? -1 : Math.addExact(currentTime, total),
191-
temp,
207+
negative ? -temp : temp,
192208
unit,
193209
timestampPrecision));
194210
temp = 0;
@@ -203,11 +219,17 @@ public static long convertDurationStrToLong(
203219

204220
public static long convertDurationStrToLong(
205221
long currentTime, String duration, boolean convertYearToMonth) {
222+
return convertDurationStrToLong(currentTime, duration, convertYearToMonth, false);
223+
}
224+
225+
public static long convertDurationStrToLong(
226+
long currentTime, String duration, boolean convertYearToMonth, boolean negative) {
206227
return convertDurationStrToLong(
207228
currentTime,
208229
duration,
209230
CommonDescriptor.getInstance().getConfig().getTimestampPrecision(),
210-
convertYearToMonth);
231+
convertYearToMonth,
232+
negative);
211233
}
212234

213235
/**

‎iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,19 @@
1919
package org.apache.iotdb.db.utils;
2020

2121
import org.apache.iotdb.commons.exception.SemanticException;
22+
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral;
2223
import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils;
2324
import org.apache.iotdb.db.protocol.session.IClientSession;
2425
import org.apache.iotdb.db.protocol.session.InternalClientSession;
2526
import org.apache.iotdb.db.protocol.session.SessionManager;
27+
import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
2628

2729
import org.apache.tsfile.utils.TimeDuration;
2830
import org.junit.Assert;
2931
import org.junit.Ignore;
3032
import org.junit.Test;
3133

34+
import java.time.Instant;
3235
import java.time.ZoneId;
3336
import java.time.ZoneOffset;
3437
import java.time.ZonedDateTime;
@@ -260,6 +263,76 @@ public void getConvertDurationIncludingMonthUnit() {
260263
}
261264
}
262265

266+
/** "- Nmo" in a date expression must go back N calendar months from the original time. */
267+
@Test
268+
public void dateExpressionSubtractNaturalMonthTest() {
269+
String[][] cases = {
270+
// expression, expected
271+
{"2024-03-15T00:00:00+00:00 - 1mo", "2024-02-15T00:00:00Z"},
272+
{"2024-02-15T00:00:00+00:00 - 1mo", "2024-01-15T00:00:00Z"},
273+
{"2025-03-15T00:00:00+00:00 - 1mo", "2025-02-15T00:00:00Z"},
274+
{"2024-03-15T00:00:00+00:00 - 2mo", "2024-01-15T00:00:00Z"},
275+
{"2024-01-15T00:00:00+00:00 - 1mo", "2023-12-15T00:00:00Z"},
276+
// month end is clamped, the same as for "+ 1mo"
277+
{"2024-03-31T00:00:00+00:00 - 1mo", "2024-02-29T00:00:00Z"},
278+
{"2025-03-31T00:00:00+00:00 - 1mo", "2025-02-28T00:00:00Z"},
279+
{"2024-05-31T00:00:00+00:00 - 1mo", "2024-04-30T00:00:00Z"},
280+
// mixed units are applied from left to right (compound durations are tree syntax only)
281+
{"2024-03-31T00:00:00+00:00 - 1mo1d", "2024-02-28T00:00:00Z", "tree"},
282+
{"2024-03-15T00:00:00+00:00 - 1w2d", "2024-03-06T00:00:00Z", "tree"},
283+
{"2024-03-31T00:00:00+00:00 - 1mo - 1d", "2024-02-28T00:00:00Z"},
284+
// negative timestamp
285+
{"1969-03-15T00:00:00+00:00 - 1mo", "1969-02-15T00:00:00Z"},
286+
// subtraction is the inverse of addition
287+
{"2024-02-15T00:00:00+00:00 + 1mo - 1mo", "2024-02-15T00:00:00Z"},
288+
// addition is unchanged
289+
{"2024-02-15T00:00:00+00:00 + 1mo", "2024-03-15T00:00:00Z"},
290+
{"2024-01-31T00:00:00+00:00 + 1mo", "2024-02-29T00:00:00Z"},
291+
};
292+
assertDateExpressions("UTC", cases);
293+
}
294+
295+
/** Natural months are counted in the session time zone, keeping the local time across DST. */
296+
@Test
297+
public void dateExpressionSubtractNaturalMonthWithTimeZoneTest() {
298+
assertDateExpressions(
299+
"America/New_York",
300+
new String[][] {
301+
{"2024-04-15T12:00:00-04:00 - 1mo", "2024-03-15T16:00:00Z"},
302+
{"2024-03-15T12:00:00-04:00 - 1mo", "2024-02-15T17:00:00Z"},
303+
});
304+
assertDateExpressions(
305+
"Asia/Kolkata",
306+
new String[][] {{"2024-03-01T00:00:00+05:30 - 1mo", "2024-01-31T18:30:00Z"}});
307+
}
308+
309+
private void assertDateExpressions(String sessionZone, String[][] cases) {
310+
ZoneId zoneId = ZoneId.of(sessionZone);
311+
IClientSession session = new InternalClientSession("dateExpressionNaturalMonthTest");
312+
session.setZoneId(zoneId);
313+
DateTimeUtils.initTimestampPrecision();
314+
SqlParser tableParser = new SqlParser();
315+
try {
316+
SessionManager.getInstance().registerSession(session);
317+
for (String[] c : cases) {
318+
long expected = Instant.parse(c[1]).toEpochMilli();
319+
Assert.assertEquals(
320+
"tree: " + c[0],
321+
expected,
322+
(long) DataNodeDateTimeUtils.parseDateTimeExpressionToLong(c[0], zoneId));
323+
if (c.length > 2) {
324+
continue;
325+
}
326+
Assert.assertEquals(
327+
"table: " + c[0],
328+
expected,
329+
((LongLiteral) tableParser.createExpression(c[0], zoneId)).getParsedValue());
330+
}
331+
} finally {
332+
SessionManager.getInstance().removeCurrSession();
333+
}
334+
}
335+
263336
public void testConvertDatetimeStrToLongWithoutMS(
264337
ZoneOffset zoneOffset, ZoneId zoneId, long res) {
265338
String[] timeFormatWithoutMs =

0 commit comments

Comments
 (0)