From 8d3cb822fd6180b4d6a338b439f647f0c734e55c Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 31 Jan 2024 04:01:20 -0800 Subject: [PATCH] [CALCITE-2067] RexBuilder can't handle NaN,Infinity double constants Signed-off-by: Mihai Budiu --- .../adapter/arrow/ArrowTranslator.java | 2 + .../adapter/arrow/ArrowAdapterTest.java | 2 +- .../calcite/rel/rel2sql/SqlImplementor.java | 2 +- .../org/apache/calcite/rex/RexBuilder.java | 38 +++++++-- .../org/apache/calcite/rex/RexLiteral.java | 79 +++++++++++++------ .../org/apache/calcite/tools/RelBuilder.java | 2 +- .../java/org/apache/calcite/util/Util.java | 19 +++++ .../rel/rel2sql/RelToSqlConverterTest.java | 6 +- .../apache/calcite/rex/RexExecutorTest.java | 6 +- .../org/apache/calcite/test/JdbcTest.java | 2 +- .../apache/calcite/test/RelOptRulesTest.java | 27 +++++++ .../org/apache/calcite/util/UtilTest.java | 27 +++++++ .../apache/calcite/test/RelOptRulesTest.xml | 64 +++++++++++---- .../test/TypeCoercionConverterTest.xml | 12 +-- core/src/test/resources/sql/cast.iq | 2 +- .../adapter/druid/DruidExpressions.java | 6 +- .../apache/calcite/test/DruidAdapter2IT.java | 16 ++-- .../apache/calcite/test/DruidAdapterIT.java | 8 +- .../calcite/piglet/PigRelExVisitor.java | 8 +- .../org/apache/calcite/test/PigRelExTest.java | 5 +- .../org/apache/calcite/test/PigRelOpTest.java | 2 +- site/_docs/history.md | 11 +++ .../org/apache/calcite/util/TestUtil.java | 7 ++ 23 files changed, 280 insertions(+), 73 deletions(-) diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java index 0015471d146b..d531e333cf18 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java @@ -228,6 +228,8 @@ private static String getLiteralType(Object literal) { } } else if (String.class.equals(literal.getClass())) { return "string"; + } else if (literal instanceof Double) { + return "float"; } throw new UnsupportedOperationException("Unsupported literal " + literal); } diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index d48576a11701..349d85a4c46b 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -373,7 +373,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select * from arrowdata\n" + " where \"floatField\"=15.0"; String plan = "PLAN=ArrowToEnumerableConverter\n" - + " ArrowFilter(condition=[=(CAST($2):DOUBLE, 15.0)])\n" + + " ArrowFilter(condition=[=(CAST($2):DOUBLE, 15.0E0)])\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=15; stringField=15; floatField=15.0; longField=15\n"; diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index 8b26d90bf775..0884ca94cbd9 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -1498,7 +1498,7 @@ public static SqlNode toSql(RexLiteral literal) { case EXACT_NUMERIC: { if (SqlTypeName.APPROX_TYPES.contains(typeName)) { return SqlLiteral.createApproxNumeric( - castNonNull(literal.getValueAs(BigDecimal.class)).toPlainString(), POS); + castNonNull(literal.getValueAs(Double.class)).toString(), POS); } else { return SqlLiteral.createExactNumeric( castNonNull(literal.getValueAs(BigDecimal.class)).toPlainString(), POS); diff --git a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java index 5cc4b2117032..3ebabd164c5e 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java @@ -1437,7 +1437,21 @@ public RexLiteral makeApproxLiteral(BigDecimal bd) { public RexLiteral makeApproxLiteral(@Nullable BigDecimal bd, RelDataType type) { assert SqlTypeFamily.APPROXIMATE_NUMERIC.getTypeNames().contains( type.getSqlTypeName()); - return makeLiteral(bd, type, SqlTypeName.DOUBLE); + return makeLiteral(bd != null ? bd.doubleValue() : null, type, SqlTypeName.DOUBLE); + } + + /** + * Creates an approximate numeric literal (double or float) + * from a Double value. + * + * @param val literal value + * @param type approximate numeric type + * @return new literal + */ + public RexLiteral makeApproxLiteral(Double val, RelDataType type) { + assert SqlTypeFamily.APPROXIMATE_NUMERIC.getTypeNames().contains( + type.getSqlTypeName()); + return makeLiteral(val, type, SqlTypeName.DOUBLE); } /** @@ -2013,7 +2027,10 @@ public RexNode makeLiteral(@Nullable Object value, RelDataType type, case FLOAT: case REAL: case DOUBLE: - return makeApproxLiteral((BigDecimal) value, type); + if (value instanceof Double) { + return makeApproxLiteral((Double) value, type); + } + return makeApproxLiteral(((BigDecimal) value).doubleValue(), type); case BOOLEAN: return (Boolean) value ? booleanTrue : booleanFalse; case TIME: @@ -2159,13 +2176,24 @@ public RexNode makeLambdaCall(RexNode expr, List parameters) { type.getSqlTypeName()); return new BigDecimal(((Number) o).longValue()); case REAL: - case FLOAT: - case DOUBLE: if (o instanceof BigDecimal) { return o; } - return new BigDecimal(((Number) o).doubleValue(), MathContext.DECIMAL64) + // Float values are stored as Doubles + if (o instanceof Float) { + return ((Float) o).doubleValue(); + } + if (o instanceof Double) { + return o; + } + return new BigDecimal(((Number) o).doubleValue(), MathContext.DECIMAL32) .stripTrailingZeros(); + case FLOAT: + case DOUBLE: + if (o instanceof Double) { + return o; + } + return ((Number) o).doubleValue(); case CHAR: case VARCHAR: if (o instanceof NlsString) { diff --git a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java index fd34c9967422..1d95b0b3eda1 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java +++ b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java @@ -55,6 +55,7 @@ import java.io.PrintWriter; import java.math.BigDecimal; +import java.math.MathContext; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.text.SimpleDateFormat; @@ -112,9 +113,11 @@ * {@link BigDecimal} * * - * {@link SqlTypeName#DOUBLE} + * {@link SqlTypeName#DOUBLE}, + * {@link SqlTypeName#REAL}, + * {@link SqlTypeName#FLOAT} * Approximate number, for example 6.023E-23. - * {@link BigDecimal} + * {@link Double}. * * * {@link SqlTypeName#DATE} @@ -193,7 +196,7 @@ public class RexLiteral extends RexNode { /** * The value of this literal. Must be consistent with its type, as per * {@link #valueMatchesType}. For example, you can't store an - * {@link Integer} value here just because you feel like it -- all numbers are + * {@link Integer} value here just because you feel like it -- all exact numbers are * represented by a {@link BigDecimal}. But since this field is private, it * doesn't really matter how the values are stored. */ @@ -204,12 +207,9 @@ public class RexLiteral extends RexNode { */ private final RelDataType type; - // TODO jvs 26-May-2006: Use SqlTypeFamily instead; it exists - // for exactly this purpose (to avoid the confusion which results - // from overloading SqlTypeName). /** * An indication of the broad type of this literal -- even if its type isn't - * a SQL type. Sometimes this will be different than the SQL type; for + * a SQL type. Sometimes this will be different from the SQL type; for * example, all exact numbers, including integers have typeName * {@link SqlTypeName#DECIMAL}. See {@link #valueMatchesType} for the * definitive story. @@ -294,10 +294,10 @@ public final String computeDigest( } /** - * Returns true if {@link RexDigestIncludeType#OPTIONAL} digest would include data type. + * Returns whether {@link RexDigestIncludeType} digest would include data type. * * @see RexCall#computeDigest(boolean) - * @return true if {@link RexDigestIncludeType#OPTIONAL} digest would include data type + * @return whether {@link RexDigestIncludeType} digest would include data type */ @RequiresNonNull("type") RexDigestIncludeType digestIncludesType( @@ -328,11 +328,12 @@ public static boolean valueMatchesType( } // fall through case DECIMAL: + case BIGINT: + return value instanceof BigDecimal; case DOUBLE: case FLOAT: case REAL: - case BIGINT: - return value instanceof BigDecimal; + return value instanceof Double; case DATE: return value instanceof DateString; case TIME: @@ -660,8 +661,14 @@ private static void appendAsJava(@Nullable Comparable value, StringBuilder sb, break; case DOUBLE: case FLOAT: - assert value instanceof BigDecimal; - sb.append(Util.toScientificNotation((BigDecimal) value)); + if (value instanceof BigDecimal) { + sb.append(Util.toScientificNotation((BigDecimal) value)); + } else { + assert value instanceof Double; + Double d = (Double) value; + String repr = Util.toScientificNotation(d); + sb.append(repr); + } break; case BIGINT: assert value instanceof BigDecimal; @@ -1075,22 +1082,48 @@ public boolean isNull() { case BIGINT: case INTEGER: case SMALLINT: - case TINYINT: - case DOUBLE: - case REAL: - case FLOAT: + case TINYINT: { + BigDecimal bd = (BigDecimal) value; if (clazz == Long.class) { - return clazz.cast(((BigDecimal) value).longValue()); + return clazz.cast(bd.longValue()); } else if (clazz == Integer.class) { - return clazz.cast(((BigDecimal) value).intValue()); + return clazz.cast(bd.intValue()); } else if (clazz == Short.class) { - return clazz.cast(((BigDecimal) value).shortValue()); + return clazz.cast(bd.shortValue()); } else if (clazz == Byte.class) { - return clazz.cast(((BigDecimal) value).byteValue()); + return clazz.cast(bd.byteValue()); } else if (clazz == Double.class) { - return clazz.cast(((BigDecimal) value).doubleValue()); + return clazz.cast(bd.doubleValue()); } else if (clazz == Float.class) { - return clazz.cast(((BigDecimal) value).floatValue()); + return clazz.cast(bd.floatValue()); + } + break; + } + case DOUBLE: + case REAL: + case FLOAT: + if (value instanceof Double) { + Double d = (Double) value; + if (clazz == Long.class) { + return clazz.cast(d.longValue()); + } else if (clazz == Integer.class) { + return clazz.cast(d.intValue()); + } else if (clazz == Short.class) { + return clazz.cast(d.shortValue()); + } else if (clazz == Byte.class) { + return clazz.cast(d.byteValue()); + } else if (clazz == Double.class) { + // Cast still needed, since the Java compiler does not understand + // that T is double. + return clazz.cast(d); + } else if (clazz == Float.class) { + return clazz.cast(d.floatValue()); + } else if (clazz == BigDecimal.class) { + // This particular conversion is lossy, since in general BigDecimal cannot + // represent accurately FP values. However, this is the best we can do. + // This conversion used to be in RexBuilder, used when creating a RexLiteral. + return clazz.cast(new BigDecimal(d, MathContext.DECIMAL64).stripTrailingZeros()); + } } break; case DATE: diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 86472f9d22d9..4c7357da8132 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -480,7 +480,7 @@ public RexLiteral literal(@Nullable Object value) { return rexBuilder.makeExactLiteral((BigDecimal) value); } else if (value instanceof Float || value instanceof Double) { return rexBuilder.makeApproxLiteral( - BigDecimal.valueOf(((Number) value).doubleValue())); + ((Number) value).doubleValue(), getTypeFactory().createSqlType(SqlTypeName.DOUBLE)); } else if (value instanceof Number) { return rexBuilder.makeExactLiteral( BigDecimal.valueOf(((Number) value).longValue())); diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java index a3fcafaa2484..b9b8e2085baf 100644 --- a/core/src/main/java/org/apache/calcite/util/Util.java +++ b/core/src/main/java/org/apache/calcite/util/Util.java @@ -534,6 +534,21 @@ public static void println( pw.println(); } + /** + * Formats a double value to a String ensuring that the output + * is in scientific notation if the value is not "special". + * (Special values include infinities and NaN.) + */ + public static String toScientificNotation(Double d) { + String repr = Double.toString(d); + if (!repr.toLowerCase(Locale.ENGLISH).contains("e") + && !d.isInfinite() + && !d.isNaN()) { + repr += "E0"; + } + return repr; + } + /** * Formats a {@link BigDecimal} value to a string in scientific notation For * example
@@ -558,6 +573,10 @@ public static String toScientificNotation(BigDecimal bd) { int len = unscaled.length(); int scale = bd.scale(); int e = len - scale - 1; + if (bd.stripTrailingZeros().equals(BigDecimal.ZERO)) { + // Without this adjustment 0.0 generates 0E-1 + e = 0; + } StringBuilder ret = new StringBuilder(); if (bd.signum() < 0) { diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 74f4450df405..6d2612b329a7 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -407,7 +407,7 @@ private static String toSql(RelNode root, SqlDialect dialect, + "where \"net_weight\" <> 10 or \"net_weight\" is null"; final String expected = "SELECT \"product_id\", \"shelf_width\"\n" + "FROM \"foodmart\".\"product\"\n" - + "WHERE \"net_weight\" <> 10 OR \"net_weight\" IS NULL"; + + "WHERE \"net_weight\" <> CAST(10 AS DOUBLE) OR \"net_weight\" IS NULL"; sql(query).ok(expected); } @@ -4391,7 +4391,7 @@ private SqlDialect nonOrdinalDialect() { + " select \"product_id\", 0 as \"net_weight\"\n" + " from \"sales_fact_1997\") t0"; final String expected = "SELECT SUM(CASE WHEN \"product_id\" = 0" - + " THEN \"net_weight\" ELSE 0 END) AS \"NET_WEIGHT\"\n" + + " THEN \"net_weight\" ELSE 0E0 END) AS \"NET_WEIGHT\"\n" + "FROM (SELECT \"product_id\", \"net_weight\"\n" + "FROM \"foodmart\".\"product\"\n" + "UNION ALL\n" @@ -6507,7 +6507,7 @@ private void checkLiteral2(String expression, String expected) { + "PATTERN (\"STRT\" \"DOWN\" + \"UP\" +)\n" + "DEFINE " + "\"DOWN\" AS PREV(\"DOWN\".\"net_weight\", 0) = " - + "0 OR PREV(\"DOWN\".\"net_weight\", 0) = 1, " + + "CAST(0 AS DOUBLE) OR PREV(\"DOWN\".\"net_weight\", 0) = CAST(1 AS DOUBLE), " + "\"UP\" AS PREV(\"UP\".\"net_weight\", 0) > " + "PREV(\"UP\".\"net_weight\", 1))"; sql(sql).ok(expected); diff --git a/core/src/test/java/org/apache/calcite/rex/RexExecutorTest.java b/core/src/test/java/org/apache/calcite/rex/RexExecutorTest.java index 8387269782b9..5586f2388fff 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexExecutorTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexExecutorTest.java @@ -389,9 +389,11 @@ interface Action { final RexCall first = (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LN, rexBuilder.makeLiteral(3, integer, true)); + // Division by zero causes an exception during evaluation final RexCall second = - (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LN, - rexBuilder.makeLiteral(-2, integer, true)); + (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.DIVIDE_INTEGER, + rexBuilder.makeLiteral(-2, integer, true), + rexBuilder.makeLiteral(0, integer, true)); executor.reduce(rexBuilder, ImmutableList.of(first, second), reducedValues); assertThat(reducedValues, hasSize(2)); diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 4eeb666d4f09..4f4838d7abaf 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -389,7 +389,7 @@ static void forEachExpand(Runnable r) { + "expr#7=[null:JavaType(class java.lang.Integer)], " + "empid=[$t3], deptno=[$t4], name=[$t5], salary=[$t6], " + "commission=[$t7])\n" - + " EnumerableValues(tuples=[[{ 'Fred', 56, 123.4 }]])\n"; + + " EnumerableValues(tuples=[[{ 'Fred', 56, 123.4000015258789E0 }]])\n"; assertThat(resultSet.getString(1), isLinux(expected)); // With named columns diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 6c672b72ab3b..cb15194c038c 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -3341,6 +3341,33 @@ private void checkPushJoinThroughUnionOnRightDoesNotMatchSemiOrAntiJoin(JoinRelT .check(); } + /** Test case for + * [CALCITE-2067] RexLiteral cannot represent accurately floating point values, + * including NaN, Infinity. */ + @Test public void testDoubleReduction() { + // Without the fix for CALCITE-2067 the result returned below is + // 1008618.49. Ironically, that result is more accurate; however + // it is not the result returned by the pow() function, which is + // 1008618.4899999999 + final String sql = "SELECT power(1004.3, 2)"; + sql(sql) + .withRule(CoreRules.PROJECT_REDUCE_EXPRESSIONS) + .check(); + } + + /** Test case for + * [CALCITE-2067] RexLiteral cannot represent accurately floating point values, + * including NaN, Infinity. */ + @Test public void testDoubleReduction2() { + // Without the fix for CALCITE-2067 the following expression is not + // reduced to Infinity, since Infinity cannot be represented + // as a BigDecimal value. + final String sql2 = "SELECT 1.0 / 0.0e0"; + sql(sql2) + .withRule(CoreRules.PROJECT_REDUCE_EXPRESSIONS) + .check(); + } + /** Tests that {@link UnionMergeRule} does nothing if its arguments have * are different set operators, {@link Union} and {@link Intersect}. */ @Test void testMergeSetOpMixed() { diff --git a/core/src/test/java/org/apache/calcite/util/UtilTest.java b/core/src/test/java/org/apache/calcite/util/UtilTest.java index fbca0845a439..94456f89ed04 100644 --- a/core/src/test/java/org/apache/calcite/util/UtilTest.java +++ b/core/src/test/java/org/apache/calcite/util/UtilTest.java @@ -108,6 +108,7 @@ import static org.apache.calcite.test.Matchers.isLinux; import static org.apache.calcite.util.ReflectUtil.isStatic; +import static org.apache.calcite.util.TestUtil.assertThatScientific; import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.anyOf; @@ -161,6 +162,10 @@ class UtilTest { @Test void testScientificNotation() { BigDecimal bd; + bd = new BigDecimal("0.0"); + TestUtil.assertEqualsVerbose( + "0E0", + Util.toScientificNotation(bd)); bd = new BigDecimal("0.001234"); TestUtil.assertEqualsVerbose( "1.234E-3", @@ -209,6 +214,28 @@ class UtilTest { Util.toScientificNotation(bd)); } + @Test void testDoubleScientificNotation() { + assertThatScientific("0.001234", is("0.001234E0")); + assertThatScientific("0.001", is("0.001E0")); + assertThatScientific("-0.001", is("-0.001E0")); + assertThatScientific("1", is("1.0E0")); + assertThatScientific("-1", is("-1.0E0")); + assertThatScientific("1.0", is("1.0E0")); + assertThatScientific("12345", is("12345.0E0")); + assertThatScientific("12345.00", is("12345.0E0")); + assertThatScientific("12345.001", is("12345.001E0")); + + // test truncate + assertThatScientific("1.23456789012345678901", is("1.2345678901234567E0")); + assertThatScientific("-1.23456789012345678901", is("-1.2345678901234567E0")); + + // special values + assertThatScientific("Infinity", is("Infinity")); + assertThatScientific("-Infinity", is("-Infinity")); + assertThatScientific("NaN", is("NaN")); + assertThatScientific("-0.0", is("-0.0E0")); + } + @Test void testToJavaId() throws UnsupportedEncodingException { assertEquals( "ID$0$foo", diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 0ebdb9b469d4..ec4d69b9c083 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -1566,7 +1566,7 @@ case when cast(ename as double) < 5 then 0.0 ($1, 'abc'), $1, null:VARCHAR(20))):DOUBLE, 5), 0.0:DOUBLE, CASE(IS NOT NULL(CAST(CASE(>($1, 'abc'), $1, null:VARCHAR(20))):DOUBLE), CAST(CAST(CASE(>($1, 'abc'), $1, null:VARCHAR(20))):DOUBLE):DOUBLE NOT NULL, 1.0:DOUBLE))]) +LogicalProject(T=[CASE(<(CAST(CASE(>($1, 'abc'), $1, null:VARCHAR(20))):DOUBLE, 5), 0.0E0:DOUBLE, CASE(IS NOT NULL(CAST(CASE(>($1, 'abc'), $1, null:VARCHAR(20))):DOUBLE), CAST(CAST(CASE(>($1, 'abc'), $1, null:VARCHAR(20))):DOUBLE):DOUBLE NOT NULL, 1.0E0:DOUBLE))]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -3026,6 +3026,40 @@ LogicalAggregate(group=[{0}], EXPR$1=[SUM($3)], EXPR$2=[MIN($4)], EXPR$3=[COUNT( LogicalFilter(condition=[false]) LogicalAggregate(group=[{}], agg#0=[COUNT()]) LogicalTableScan(table=[[scott, EMP]]) +]]> + + + + + + + + + + + + + + + + + + + + + + @@ -12257,13 +12291,13 @@ from emp]]> @@ -16484,9 +16518,9 @@ LogicalProject(DEPTNO=[$0], NAME=[$1]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalAggregate(group=[{0}]) LogicalUnion(all=[true]) - LogicalValues(tuples=[[{ 12 }]]) - LogicalValues(tuples=[[{ 34 }]]) - LogicalValues(tuples=[[{ 56.4 }]]) + LogicalValues(tuples=[[{ 12.0E0 }]]) + LogicalValues(tuples=[[{ 34.0E0 }]]) + LogicalValues(tuples=[[{ 56.4E0 }]]) ]]> @@ -16496,7 +16530,7 @@ LogicalProject(DEPTNO=[$0], NAME=[$1]) LogicalProject(DEPTNO=[$0], NAME=[$1], DEPTNO0=[CAST($0):DOUBLE NOT NULL]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalAggregate(group=[{0}]) - LogicalValues(tuples=[[{ 12 }, { 34 }, { 56.4 }]]) + LogicalValues(tuples=[[{ 12.0E0 }, { 34.0E0 }, { 56.4E0 }]]) ]]> @@ -16514,15 +16548,15 @@ LogicalProject(EXPR$0=[CAST(OR(AND(IS NOT NULL($6), <>($2, 0)), AND(<($3, $2), n LogicalAggregate(group=[{}], agg#0=[COUNT()], agg#1=[COUNT($0)]) LogicalProject(EXPR$0=[$0], $f1=[true]) LogicalUnion(all=[true]) - LogicalValues(tuples=[[{ 12 }]]) - LogicalValues(tuples=[[{ 34 }]]) - LogicalValues(tuples=[[{ 56.4 }]]) + LogicalValues(tuples=[[{ 12.0E0 }]]) + LogicalValues(tuples=[[{ 34.0E0 }]]) + LogicalValues(tuples=[[{ 56.4E0 }]]) LogicalAggregate(group=[{0}], agg#0=[MIN($1)]) LogicalProject(EXPR$0=[$0], $f1=[true]) LogicalUnion(all=[true]) - LogicalValues(tuples=[[{ 12 }]]) - LogicalValues(tuples=[[{ 34 }]]) - LogicalValues(tuples=[[{ 56.4 }]]) + LogicalValues(tuples=[[{ 12.0E0 }]]) + LogicalValues(tuples=[[{ 34.0E0 }]]) + LogicalValues(tuples=[[{ 56.4E0 }]]) ]]> @@ -16534,10 +16568,10 @@ LogicalProject(EXPR$0=[CAST(OR(AND(IS NOT NULL($6), <>($2, 0)), AND(<($3, $2), n LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalAggregate(group=[{}], agg#0=[COUNT()], agg#1=[COUNT($0)]) LogicalProject(EXPR$0=[$0], $f1=[true]) - LogicalValues(tuples=[[{ 12 }, { 34 }, { 56.4 }]]) + LogicalValues(tuples=[[{ 12.0E0 }, { 34.0E0 }, { 56.4E0 }]]) LogicalAggregate(group=[{0}], agg#0=[MIN($1)]) LogicalProject(EXPR$0=[$0], $f1=[true]) - LogicalValues(tuples=[[{ 12 }, { 34 }, { 56.4 }]]) + LogicalValues(tuples=[[{ 12.0E0 }, { 34.0E0 }, { 56.4E0 }]]) ]]> diff --git a/core/src/test/resources/org/apache/calcite/test/TypeCoercionConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/TypeCoercionConverterTest.xml index c88d38118c0c..c5476150a704 100644 --- a/core/src/test/resources/org/apache/calcite/test/TypeCoercionConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/TypeCoercionConverterTest.xml @@ -158,11 +158,11 @@ LogicalTableModify(table=[[CATALOG, SALES, T1]], operation=[INSERT], flattened=[ LogicalUnion(all=[false]) LogicalUnion(all=[false]) LogicalUnion(all=[false]) - LogicalValues(tuples=[[{ 'a', 1, 1, 0, 0, 0, 0, 2021-11-28 00:00:00, 2021-11-28, X'0a', false }]]) - LogicalValues(tuples=[[{ 'b', 2, 2, 0, 0, 0, 0, 2021-11-28 00:00:00, 2021-11-28, X'0a', false }]]) - LogicalValues(tuples=[[{ 'c', 3, 3, 0, 0, 0, 0, 2021-11-28 00:00:00, 2021-11-28, X'0a', false }]]) - LogicalValues(tuples=[[{ 'd', 4, 4, 0, 0, 0, 0, 2021-11-28 00:00:00, 2021-11-28, X'0a', false }]]) - LogicalValues(tuples=[[{ 'e', 5, 5, 0, 0, 0, 0, 2021-11-28 00:00:00, 2021-11-28, X'0a', false }]]) + LogicalValues(tuples=[[{ 'a', 1, 1, 0, 0.0E0, 0.0E0, 0, 2021-11-28 00:00:00, 2021-11-28, X'0a', false }]]) + LogicalValues(tuples=[[{ 'b', 2, 2, 0, 0.0E0, 0.0E0, 0, 2021-11-28 00:00:00, 2021-11-28, X'0a', false }]]) + LogicalValues(tuples=[[{ 'c', 3, 3, 0, 0.0E0, 0.0E0, 0, 2021-11-28 00:00:00, 2021-11-28, X'0a', false }]]) + LogicalValues(tuples=[[{ 'd', 4, 4, 0, 0.0E0, 0.0E0, 0, 2021-11-28 00:00:00, 2021-11-28, X'0a', false }]]) + LogicalValues(tuples=[[{ 'e', 5, 5, 0, 0.0E0, 0.0E0, 0, 2021-11-28 00:00:00, 2021-11-28, X'0a', false }]]) ]]> @@ -173,7 +173,7 @@ LogicalTableModify(table=[[CATALOG, SALES, T1]], operation=[INSERT], flattened=[ diff --git a/core/src/test/resources/sql/cast.iq b/core/src/test/resources/sql/cast.iq index b615bc7affd8..a24e1a42345e 100644 --- a/core/src/test/resources/sql/cast.iq +++ b/core/src/test/resources/sql/cast.iq @@ -131,7 +131,7 @@ values cast('-123.45' as double); (1 row) !ok -EnumerableValues(tuples=[[{ -1.2345E2 }]]) +EnumerableValues(tuples=[[{ -123.45E0 }]]) !plan values cast('false' as boolean); diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidExpressions.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidExpressions.java index 73045d208cea..cbdf146ea1d1 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidExpressions.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidExpressions.java @@ -33,6 +33,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -136,8 +137,11 @@ private DruidExpressions() { // deal with this for now return null; } else if (SqlTypeName.NUMERIC_TYPES.contains(sqlTypeName)) { + // This conversion is lossy for Double values. + // However, Druid does not support floating point literal values + // if they are formatted using scientific notation. return DruidExpressions.numberLiteral( - requireNonNull((Number) RexLiteral.value(rexNode))); + requireNonNull((RexLiteral) rexNode).getValueAs(BigDecimal.class)); } else if (SqlTypeFamily.INTERVAL_DAY_TIME == sqlTypeName.getFamily()) { // Calcite represents DAY-TIME intervals in milliseconds. final long milliseconds = diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java index 8c522b06a0bf..a44add809402 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java @@ -2634,7 +2634,10 @@ private void testCountWithApproxDistinct(boolean approx, String sql, + "from \"foodmart\" " + "where cast(\"product_id\" as double) = 1016.0"; final String plan = "PLAN=EnumerableInterpreter\n" - + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], filter=[=(CAST($1):DOUBLE, 1016.0)], projects=[[$91]], groups=[{}], aggs=[[SUM($0)]])"; + + " DruidQuery(table=[[foodmart, foodmart]], " + + "intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], " + + "filter=[=(CAST($1):DOUBLE, 1016.0E0)], " + + "projects=[[$91]], groups=[{}], aggs=[[SUM($0)]])"; final String druidQuery = "{'queryType':'timeseries','dataSource':'foodmart','descending':false,'granularity':'all'," + "'filter':{'type':'bound','dimension':'product_id','lower':'1016.0'," @@ -2660,7 +2663,10 @@ private void testCountWithApproxDistinct(boolean approx, String sql, + "from \"foodmart\" " + "where cast(\"product_id\" as double) <> 1016.0"; final String plan = "PLAN=EnumerableInterpreter\n" - + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], filter=[<>(CAST($1):DOUBLE, 1016.0)], projects=[[$91]], groups=[{}], aggs=[[SUM($0)]])"; + + " DruidQuery(table=[[foodmart, foodmart]], " + + "intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], " + + "filter=[<>(CAST($1):DOUBLE, 1016.0E0)], " + + "projects=[[$91]], groups=[{}], aggs=[[SUM($0)]])"; final String druidQuery = "{'queryType':'timeseries','dataSource':'foodmart','descending':false,'granularity':'all'," + "'filter':{'type':'not','field':{'type':'bound','dimension':'product_id','" @@ -3078,7 +3084,7 @@ private void testCountWithApproxDistinct(boolean approx, String sql, final String plan = "PLAN=EnumerableInterpreter\n" + " DruidQuery(table=[[foodmart, foodmart]], " + "intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]]," - + " filter=[=(FLOOR($90), 23)], groups=[{}], aggs=[[COUNT()]]"; + + " filter=[=(FLOOR($90), 23.0E0)], groups=[{}], aggs=[[COUNT()]]"; sql(sql) .returnsOrdered("EXPR$0=2") .explainContains(plan) @@ -3299,8 +3305,8 @@ private void testCountWithApproxDistinct(boolean approx, String sql, .returnsOrdered("EXPR$0=2") .explainContains("PLAN=EnumerableInterpreter\n" + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00" - + ".000Z/2992-01-10T00:00:00.000Z]], filter=[AND(>(SIN($91), 9.129452507276277E-1), >" - + "(COS($90), 4.08082061813392E-1), =(FLOOR(TAN($91)), 2), <(ABS(-(TAN($91), /(SIN" + + ".000Z/2992-01-10T00:00:00.000Z]], filter=[AND(>(SIN($91), 0.9129452507276277E0), >" + + "(COS($90), 0.40808206181339196E0), =(FLOOR(TAN($91)), 2.0E0), <(ABS(-(TAN($91), /(SIN" + "($91), COS($91)))), 1.0E-6))], groups=[{}], aggs=[[COUNT()]])"); } diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java index 200c93efe598..9e140f329373 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java @@ -3124,7 +3124,7 @@ private void testCountWithApproxDistinct(boolean approx, String sql, String expe + "where cast(\"product_id\" as double) = 1016.0"; final String plan = "PLAN=EnumerableInterpreter\n" + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], " - + "filter=[=(CAST($1):DOUBLE, 1016.0)], projects=[[$91]], groups=[{}], aggs=[[SUM($0)]])"; + + "filter=[=(CAST($1):DOUBLE, 1016.0E0)], projects=[[$91]], groups=[{}], aggs=[[SUM($0)]])"; final String druidQuery = "{'queryType':'timeseries','dataSource':'foodmart','descending':false,'granularity':'all'," + "'filter':{'type':'bound','dimension':'product_id','lower':'1016.0'," @@ -3151,7 +3151,7 @@ private void testCountWithApproxDistinct(boolean approx, String sql, String expe + "where cast(\"product_id\" as double) <> 1016.0"; final String plan = "PLAN=EnumerableInterpreter\n" + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], " - + "filter=[<>(CAST($1):DOUBLE, 1016.0)], projects=[[$91]], groups=[{}], aggs=[[SUM($0)]])"; + + "filter=[<>(CAST($1):DOUBLE, 1016.0E0)], projects=[[$91]], groups=[{}], aggs=[[SUM($0)]])"; final String druidQuery = "{'queryType':'timeseries','dataSource':'foodmart','descending':false,'granularity':'all'," + "'filter':{'type':'not','field':{'type':'bound','dimension':'product_id','" @@ -3725,7 +3725,7 @@ private void testCountWithApproxDistinct(boolean approx, String sql, String expe final String plan = "PLAN=EnumerableInterpreter\n" + " DruidQuery(table=[[foodmart, foodmart]], " + "intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]]," - + " filter=[=(FLOOR($90), 23)], groups=[{}], aggs=[[COUNT()]]"; + + " filter=[=(FLOOR($90), 23.0E0)], groups=[{}], aggs=[[COUNT()]]"; sql(sql, FOODMART) .returnsOrdered("EXPR$0=2") .explainContains(plan) @@ -3957,7 +3957,7 @@ private void testCountWithApproxDistinct(boolean approx, String sql, String expe .explainContains("PLAN=EnumerableInterpreter\n" + " DruidQuery(table=[[foodmart, foodmart]], " + "intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], " - + "filter=[AND(>(SIN($91), 9.129452507276277E-1), >(COS($90), 4.08082061813392E-1), =(FLOOR(TAN($91)), 2), " + + "filter=[AND(>(SIN($91), 0.9129452507276277E0), >(COS($90), 0.40808206181339196E0), =(FLOOR(TAN($91)), 2.0E0), " + "<(ABS(-(TAN($91), /(SIN($91), COS($91)))), 1.0E-6))], " + "groups=[{}], aggs=[[COUNT()]])"); } diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelExVisitor.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelExVisitor.java index c00289f7280e..49a748e5b22d 100644 --- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelExVisitor.java +++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelExVisitor.java @@ -256,8 +256,12 @@ private ImmutableList buildBinaryOperands() { final RexNode operand = stack.pop(); if (operand instanceof RexLiteral) { final Comparable value = ((RexLiteral) operand).getValue(); - assert value instanceof BigDecimal; - stack.push(builder.literal(((BigDecimal) value).negate())); + if (value instanceof BigDecimal) { + stack.push(builder.literal(((BigDecimal) value).negate())); + } else { + assert value instanceof Double; + stack.push(builder.literal(- (Double) value)); + } } else { stack.push(builder.call(SqlStdOperatorTable.UNARY_MINUS, operand)); } diff --git a/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java b/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java index b36f684fa59e..5d56694c04cb 100644 --- a/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java +++ b/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java @@ -79,7 +79,10 @@ private void checkType(String pigExpr, Matcher rowTypeMatcher) { } @Test void testConstantFloat() { - checkTranslation(".1E6 == -2.3", inTree("=(1E5:DOUBLE, -2.3:DECIMAL(2, 1))")); + // Add a variable d in the expression to prevent it from being simplified to "false". + checkTranslation(".1E6 == -2.3 + d", + // Validator converts -2.3 from DECIMAL to DOUBLE + inTree("=(100000.0E0, +(-2.3E0, $3))")); } @Test void testConstantString() { diff --git a/piglet/src/test/java/org/apache/calcite/test/PigRelOpTest.java b/piglet/src/test/java/org/apache/calcite/test/PigRelOpTest.java index 9b5709184a46..95de883788d7 100644 --- a/piglet/src/test/java/org/apache/calcite/test/PigRelOpTest.java +++ b/piglet/src/test/java/org/apache/calcite/test/PigRelOpTest.java @@ -227,7 +227,7 @@ private Fluent pig(String script) { + "A = LOAD 'scott.DEPT' as (DEPTNO:int, DNAME:chararray, LOC:CHARARRAY);\n" + "B = SAMPLE A 0.5;\n"; final String plan = "" - + "LogicalFilter(condition=[<(RAND(), 5E-1)])\n" + + "LogicalFilter(condition=[<(RAND(), 0.5E0)])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n"; final String sql = "" + "SELECT *\n" diff --git a/site/_docs/history.md b/site/_docs/history.md index e341ac57ec7c..e395d8b6abbf 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -53,6 +53,17 @@ using JDK/OpenJDK versions 8 to 23; Guava versions 21.0 to 33.3.0-jre; other software versions as specified in gradle.properties. +* [a ] + **RexLiteral cannot represent accurately floating point values, + including NaN, Infinity**. This fix changes the way RexLiteral + represents floating point values. Previously floating point values + were encoded into BigDecimal values. This caused precision loss + when representing the results of simplifying expressions whose + results are floating point. With this change RexLiteral uses + internally a Java Double value to represent a SQL DOUBLE, FLOAT, or + REAL value. The result of RexLiteral.getValue() accordingly changes + type in this case. + #### New features {: #new-features-1-38-0} diff --git a/testkit/src/main/java/org/apache/calcite/util/TestUtil.java b/testkit/src/main/java/org/apache/calcite/util/TestUtil.java index 4987235f1923..3c2917ffea04 100644 --- a/testkit/src/main/java/org/apache/calcite/util/TestUtil.java +++ b/testkit/src/main/java/org/apache/calcite/util/TestUtil.java @@ -35,8 +35,10 @@ import static org.apache.calcite.util.Util.first; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.fail; +import static java.lang.Double.parseDouble; import static java.lang.Integer.parseInt; import static java.util.Objects.requireNonNull; @@ -93,6 +95,11 @@ public static void assertEqualsVerbose( + toJavaString(actual) + '\n'); } + public static void assertThatScientific(String value, org.hamcrest.Matcher matcher) { + double d = parseDouble(value); + assertThat(Util.toScientificNotation(d), matcher); + } + /** * Converts a string (which may contain quotes and newlines) into a java * literal.