Skip to content

Commit

Permalink
Add numeric integer to decimal coercion for hive tables
Browse files Browse the repository at this point in the history
  • Loading branch information
@findinpath @losipiuk
findinpath authored and losipiuk committed Dec 7, 2023
1 parent 22b3d03 commit a0f8178
Show file tree
Hide file tree
Showing 9 changed files with 291 additions and 14 deletions.
8 changes: 4 additions & 4 deletions docs/src/main/sphinx/connector/hive.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -644,13 +644,13 @@ type conversions.
* - `CHAR`
- narrowing conversions for `CHAR`
* - `TINYINT`
- `VARCHAR`, `SMALLINT`, `INTEGER`, `BIGINT`, `DOUBLE`
- `VARCHAR`, `SMALLINT`, `INTEGER`, `BIGINT`, `DOUBLE`, `DECIMAL`
* - `SMALLINT`
- `VARCHAR`, `INTEGER`, `BIGINT`, `DOUBLE`
- `VARCHAR`, `INTEGER`, `BIGINT`, `DOUBLE`, `DECIMAL`
* - `INTEGER`
- `VARCHAR`, `BIGINT`, `DOUBLE`
- `VARCHAR`, `BIGINT`, `DOUBLE`, `DECIMAL`
* - `BIGINT`
- `VARCHAR`, `DOUBLE`
- `VARCHAR`, `DOUBLE`, `DECIMAL`
* - `REAL`
- `DOUBLE`, `DECIMAL`
* - `DOUBLE`
Expand Down
19 changes: 17 additions & 2 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/CoercionUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
import static io.trino.plugin.hive.coercions.DecimalCoercers.createDecimalToRealCoercer;
import static io.trino.plugin.hive.coercions.DecimalCoercers.createDecimalToVarcharCoercer;
import static io.trino.plugin.hive.coercions.DecimalCoercers.createDoubleToDecimalCoercer;
import static io.trino.plugin.hive.coercions.DecimalCoercers.createIntegerNumberToDecimalCoercer;
import static io.trino.plugin.hive.coercions.DecimalCoercers.createRealToDecimalCoercer;
import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
import static io.trino.spi.block.ColumnarArray.toColumnarArray;
Expand Down Expand Up @@ -139,6 +140,9 @@ public static Type createTypeFromCoercer(TypeManager typeManager, HiveType fromH
if (toHiveType.equals(HIVE_DOUBLE)) {
return Optional.of(new IntegerNumberToDoubleCoercer<>(fromType));
}
if (toType instanceof DecimalType toDecimalType) {
return Optional.of(createIntegerNumberToDecimalCoercer(fromType, toDecimalType));
}
}
if (fromHiveType.equals(HIVE_SHORT)) {
if (toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG)) {
Expand All @@ -147,6 +151,9 @@ public static Type createTypeFromCoercer(TypeManager typeManager, HiveType fromH
if (toHiveType.equals(HIVE_DOUBLE)) {
return Optional.of(new IntegerNumberToDoubleCoercer<>(fromType));
}
if (toType instanceof DecimalType toDecimalType) {
return Optional.of(createIntegerNumberToDecimalCoercer(fromType, toDecimalType));
}
}
if (fromHiveType.equals(HIVE_INT)) {
if (toHiveType.equals(HIVE_LONG)) {
Expand All @@ -155,9 +162,17 @@ public static Type createTypeFromCoercer(TypeManager typeManager, HiveType fromH
if (toHiveType.equals(HIVE_DOUBLE)) {
return Optional.of(new IntegerNumberToDoubleCoercer<>(fromType));
}
if (toType instanceof DecimalType toDecimalType) {
return Optional.of(createIntegerNumberToDecimalCoercer(fromType, toDecimalType));
}
}
if (fromHiveType.equals(HIVE_LONG) && toHiveType.equals(HIVE_DOUBLE)) {
return Optional.of(new IntegerNumberToDoubleCoercer<>(fromType));
if (fromHiveType.equals(HIVE_LONG)) {
if (toHiveType.equals(HIVE_DOUBLE)) {
return Optional.of(new IntegerNumberToDoubleCoercer<>(fromType));
}
if (toType instanceof DecimalType toDecimalType) {
return Optional.of(createIntegerNumberToDecimalCoercer(fromType, toDecimalType));
}
}
if (fromHiveType.equals(HIVE_FLOAT) && toHiveType.equals(HIVE_DOUBLE)) {
return Optional.of(new FloatToDoubleCoercer());
Expand Down
55 changes: 55 additions & 0 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/DecimalCoercers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import io.trino.spi.type.Type;
import io.trino.spi.type.VarcharType;

import java.math.BigDecimal;

import static io.trino.spi.StandardErrorCode.INVALID_ARGUMENTS;
import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
import static io.trino.spi.type.BigintType.BIGINT;
Expand All @@ -41,6 +43,9 @@
import static io.trino.spi.type.DecimalConversions.shortToLongCast;
import static io.trino.spi.type.DecimalConversions.shortToShortCast;
import static io.trino.spi.type.Decimals.longTenToNth;
import static io.trino.spi.type.Decimals.overflows;
import static io.trino.spi.type.Decimals.writeBigDecimal;
import static io.trino.spi.type.Decimals.writeShortDecimal;
import static io.trino.spi.type.DoubleType.DOUBLE;
import static io.trino.spi.type.IntegerType.INTEGER;
import static io.trino.spi.type.RealType.REAL;
Expand Down Expand Up @@ -465,4 +470,54 @@ protected void applyCoercedValue(BlockBuilder blockBuilder, Block block, int pos
realToLongDecimal(fromType.getFloat(block, position), toType.getPrecision(), toType.getScale()));
}
}

public static <F extends Type> TypeCoercer<F, DecimalType> createIntegerNumberToDecimalCoercer(F fromType, DecimalType toType)
{
if (toType.isShort()) {
return new IntegerNumberToShortDecimalCoercer<>(fromType, toType);
}
return new IntegerNumberToLongDecimalCoercer<>(fromType, toType);
}

private static class IntegerNumberToShortDecimalCoercer<F extends Type>
extends TypeCoercer<F, DecimalType>
{
public IntegerNumberToShortDecimalCoercer(F fromType, DecimalType toType)
{
super(fromType, toType);
}

@Override
protected void applyCoercedValue(BlockBuilder blockBuilder, Block block, int position)
{
BigDecimal bigDecimal = BigDecimal.valueOf(fromType.getLong(block, position)).setScale(toType.getScale());
if (overflows(bigDecimal, toType.getPrecision())) {
blockBuilder.appendNull();
}
else {
writeShortDecimal(blockBuilder, bigDecimal.unscaledValue().longValueExact());
}
}
}

private static class IntegerNumberToLongDecimalCoercer<F extends Type>
extends TypeCoercer<F, DecimalType>
{
public IntegerNumberToLongDecimalCoercer(F fromType, DecimalType toType)
{
super(fromType, toType);
}

@Override
protected void applyCoercedValue(BlockBuilder blockBuilder, Block block, int position)
{
BigDecimal bigDecimal = BigDecimal.valueOf(fromType.getLong(block, position)).setScale(toType.getScale());
if (overflows(bigDecimal, toType.getPrecision())) {
blockBuilder.appendNull();
}
else {
writeBigDecimal(toType, blockBuilder, bigDecimal);
}
}
}
}
16 changes: 16 additions & 0 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import io.trino.plugin.hive.coercions.TypeCoercer;
import io.trino.plugin.hive.coercions.VarcharToDoubleCoercer;
import io.trino.spi.type.DateType;
import io.trino.spi.type.DecimalType;
import io.trino.spi.type.DoubleType;
import io.trino.spi.type.TimestampType;
import io.trino.spi.type.Type;
Expand All @@ -41,6 +42,7 @@
import static io.trino.orc.metadata.OrcType.OrcTypeKind.STRING;
import static io.trino.orc.metadata.OrcType.OrcTypeKind.TIMESTAMP;
import static io.trino.orc.metadata.OrcType.OrcTypeKind.VARCHAR;
import static io.trino.plugin.hive.coercions.DecimalCoercers.createIntegerNumberToDecimalCoercer;
import static io.trino.spi.type.BigintType.BIGINT;
import static io.trino.spi.type.IntegerType.INTEGER;
import static io.trino.spi.type.SmallintType.SMALLINT;
Expand Down Expand Up @@ -98,6 +100,20 @@ private OrcTypeTranslator() {}
return Optional.of(new IntegerNumberToDoubleCoercer<>(BIGINT));
}
}
if (toTrinoType instanceof DecimalType decimalType) {
if (fromOrcType == BYTE) {
return Optional.of(createIntegerNumberToDecimalCoercer(TINYINT, decimalType));
}
if (fromOrcType == SHORT) {
return Optional.of(createIntegerNumberToDecimalCoercer(SMALLINT, decimalType));
}
if (fromOrcType == INT) {
return Optional.of(createIntegerNumberToDecimalCoercer(INTEGER, decimalType));
}
if (fromOrcType == LONG) {
return Optional.of(createIntegerNumberToDecimalCoercer(BIGINT, decimalType));
}
}
return Optional.empty();
}

Expand Down
8 changes: 4 additions & 4 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveCoercionPolicy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,16 +87,16 @@ private boolean canCoerce(HiveType fromHiveType, HiveType toHiveType, HiveTimest
return fromHiveType.equals(HIVE_TIMESTAMP);
}
if (fromHiveType.equals(HIVE_BYTE)) {
return toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG) || toHiveType.equals(HIVE_DOUBLE);
return toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG) || toHiveType.equals(HIVE_DOUBLE) || toType instanceof DecimalType;
}
if (fromHiveType.equals(HIVE_SHORT)) {
return toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG) || toHiveType.equals(HIVE_DOUBLE);
return toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG) || toHiveType.equals(HIVE_DOUBLE) || toType instanceof DecimalType;
}
if (fromHiveType.equals(HIVE_INT)) {
return toHiveType.equals(HIVE_LONG) || toHiveType.equals(HIVE_DOUBLE);
return toHiveType.equals(HIVE_LONG) || toHiveType.equals(HIVE_DOUBLE) || toType instanceof DecimalType;
}
if (fromHiveType.equals(HIVE_LONG)) {
return toHiveType.equals(HIVE_DOUBLE);
return toHiveType.equals(HIVE_DOUBLE) || toType instanceof DecimalType;
}
if (fromHiveType.equals(HIVE_FLOAT)) {
return toHiveType.equals(HIVE_DOUBLE) || toType instanceof DecimalType;
Expand Down
87 changes: 83 additions & 4 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestDecimalCoercers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,17 @@
import io.trino.spi.block.Block;
import io.trino.spi.type.DecimalParseResult;
import io.trino.spi.type.Decimals;
import io.trino.spi.type.Int128;
import io.trino.spi.type.Type;
import org.junit.jupiter.api.Test;

import static io.trino.plugin.hive.HiveTimestampPrecision.NANOSECONDS;
import static io.trino.plugin.hive.HiveTimestampPrecision.DEFAULT_PRECISION;
import static io.trino.plugin.hive.HiveType.toHiveType;
import static io.trino.plugin.hive.coercions.CoercionUtils.createCoercer;
import static io.trino.spi.predicate.Utils.blockToNativeValue;
import static io.trino.spi.predicate.Utils.nativeValueToBlock;
import static io.trino.spi.type.BigintType.BIGINT;
import static io.trino.spi.type.DecimalType.createDecimalType;
import static io.trino.spi.type.IntegerType.INTEGER;
import static io.trino.spi.type.SmallintType.SMALLINT;
import static io.trino.spi.type.TinyintType.TINYINT;
Expand Down Expand Up @@ -73,12 +75,89 @@ private void testDecimalToIntCoercion(String decimalString, Type coercedType, Ob
else {
assertThat(parseResult.getType().isShort()).isTrue();
}
assertDecimalToIntCoercion(parseResult.getType(), parseResult.getObject(), coercedType, expectedValue);
assertCoercion(parseResult.getType(), parseResult.getObject(), coercedType, expectedValue);
}

private void assertDecimalToIntCoercion(Type fromType, Object valueToBeCoerced, Type toType, Object expectedValue)
@Test
public void testTinyintToDecimalCoercion()
{
// Short decimal coercion
assertCoercion(TINYINT, 12L, createDecimalType(10), 12L);
assertCoercion(TINYINT, 12L, createDecimalType(10, 2), 1_200L);
assertCoercion(TINYINT, 12L, createDecimalType(10, 5), 1_200_000L);
// Long decimal coercion
assertCoercion(TINYINT, 0L, createDecimalType(), Int128.ZERO);
assertCoercion(TINYINT, 0L, createDecimalType(), Int128.ZERO);
assertCoercion(TINYINT, 12L, createDecimalType(), Int128.valueOf(12));
assertCoercion(TINYINT, -12L, createDecimalType(), Int128.valueOf(-12));
assertCoercion(TINYINT, (long) Byte.MAX_VALUE, createDecimalType(), Int128.valueOf(Byte.MAX_VALUE));
assertCoercion(TINYINT, (long) Byte.MIN_VALUE, createDecimalType(), Int128.valueOf(Byte.MIN_VALUE));
assertCoercion(TINYINT, 12L, createDecimalType(20, 10), Int128.valueOf("120000000000"));
// Coercion overflow
assertCoercion(TINYINT, 42L, createDecimalType(6, 5), null);
}

@Test
public void testSmallintToDecimalCoercion()
{
// Short decimal coercion
assertCoercion(SMALLINT, 12L, createDecimalType(10), 12L);
assertCoercion(SMALLINT, 12L, createDecimalType(10, 2), 1_200L);
assertCoercion(SMALLINT, 12L, createDecimalType(10, 5), 1_200_000L);
// Long decimal coercion
assertCoercion(SMALLINT, 12L, createDecimalType(20, 10), Int128.valueOf("120000000000"));
assertCoercion(SMALLINT, 0L, createDecimalType(), Int128.ZERO);
assertCoercion(SMALLINT, 128L, createDecimalType(), Int128.valueOf(128));
assertCoercion(SMALLINT, -128L, createDecimalType(), Int128.valueOf(-128));
assertCoercion(SMALLINT, (long) Short.MAX_VALUE, createDecimalType(), Int128.valueOf(Short.MAX_VALUE));
assertCoercion(SMALLINT, (long) Short.MIN_VALUE, createDecimalType(), Int128.valueOf(Short.MIN_VALUE));
// Coercion overflow
assertCoercion(SMALLINT, 128L, createDecimalType(7, 5), null);
assertCoercion(SMALLINT, 128L, createDecimalType(20, 18), null);
}

@Test
public void testIntToDecimalCoercion()
{
// Short decimal coercion
assertCoercion(INTEGER, 123_456L, createDecimalType(10), 123_456L);
assertCoercion(INTEGER, 123_456L, createDecimalType(10, 3), 123_456_000L);
// Long decimal coercion
assertCoercion(INTEGER, 0L, createDecimalType(), Int128.ZERO);
assertCoercion(INTEGER, 128L, createDecimalType(), Int128.valueOf(128));
assertCoercion(INTEGER, -128L, createDecimalType(), Int128.valueOf(-128));
assertCoercion(INTEGER, (long) Integer.MAX_VALUE, createDecimalType(), Int128.valueOf(Integer.MAX_VALUE));
assertCoercion(INTEGER, (long) Integer.MIN_VALUE, createDecimalType(), Int128.valueOf(Integer.MIN_VALUE));
assertCoercion(INTEGER, 123_456L, createDecimalType(20, 10), Int128.valueOf("1234560000000000"));
// Coercion overflow
assertCoercion(INTEGER, 123_456_789L, createDecimalType(10, 5), null);
assertCoercion(INTEGER, 123_456_789L, createDecimalType(20, 13), null);
}

@Test
public void testBigintToDecimalCoercion()
{
// Short decimal coercion
assertCoercion(BIGINT, 0L, createDecimalType(10), 0L);
assertCoercion(BIGINT, 123_456_789L, createDecimalType(12), 123_456_789L);
assertCoercion(BIGINT, 123_456_789L, createDecimalType(12, 3), 123_456_789_000L);
// Long decimal coercion
assertCoercion(BIGINT, 0L, createDecimalType(), Int128.ZERO);
assertCoercion(BIGINT, 128L, createDecimalType(), Int128.valueOf(128));
assertCoercion(BIGINT, -128L, createDecimalType(), Int128.valueOf(-128));
assertCoercion(BIGINT, Long.MAX_VALUE, createDecimalType(), Int128.valueOf(Long.MAX_VALUE));
assertCoercion(BIGINT, Long.MIN_VALUE, createDecimalType(), Int128.valueOf(Long.MIN_VALUE));
assertCoercion(BIGINT, 123_456_789L, createDecimalType(20, 5), Int128.valueOf("12345678900000"));
assertCoercion(BIGINT, 123_456_789L, createDecimalType(20, 10), Int128.valueOf("1234567890000000000"));
assertCoercion(BIGINT, Long.MAX_VALUE, createDecimalType(38, 2), Int128.valueOf("922337203685477580700"));
// Coercion overflow
assertCoercion(BIGINT, 123_456_789L, createDecimalType(10, 5), null);
assertCoercion(BIGINT, Long.MAX_VALUE, createDecimalType(25, 8), null);
}

private static void assertCoercion(Type fromType, Object valueToBeCoerced, Type toType, Object expectedValue)
{
Block coercedValue = createCoercer(TESTING_TYPE_MANAGER, toHiveType(fromType), toHiveType(toType), new CoercionUtils.CoercionContext(NANOSECONDS, false)).orElseThrow()
Block coercedValue = createCoercer(TESTING_TYPE_MANAGER, toHiveType(fromType), toHiveType(toType), new CoercionUtils.CoercionContext(DEFAULT_PRECISION, true)).orElseThrow()
.apply(nativeValueToBlock(fromType, valueToBeCoerced));
assertThat(blockToNativeValue(toType, coercedValue))
.isEqualTo(expectedValue);
Expand Down
Loading

0 comments on commit a0f8178

Please sign in to comment.