Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CARBONDATA-4291] Carbon hive table supports float datatype #4231

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[CARBONDATA-4291] Carbon hive table supports float datatype #4231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,8 @@ public Object getData(int rowId) {
return getInt(rowId);
} else if (dataType == LONG) {
return getLong(rowId);
} else if (dataType == DataTypes.FLOAT) {
return getFloat(rowId);
} else if (dataType == DataTypes.DOUBLE) {
return getDouble(rowId);
} else if (DataTypes.isDecimal(dataType)) {
Expand Down Expand Up @@ -560,6 +562,8 @@ private Object getNull(int rowId) {
result = getInt(rowId);
} else if (dataType == LONG) {
result = getLong(rowId);
} else if (dataType == DataTypes.FLOAT) {
return getFloat(rowId);
} else if (dataType == DataTypes.DOUBLE) {
result = getDouble(rowId);
} else if (DataTypes.isDecimal(dataType)) {
Expand Down
6 changes: 6 additions & 0 deletions core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPageValueConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ public interface ColumnPageValueConverter {

long decodeLong(int value);

float decodeFloat(byte value);

float decodeFloat(short value);

float decodeFloat(int value);

double decodeDouble(byte value);

double decodeDouble(short value);
Expand Down
17 changes: 16 additions & 1 deletion core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,22 @@ public double getDouble(int rowId) {

@Override
public float getFloat(int rowId) {
return (float) getDouble(rowId);
DataType dataType = columnPage.getDataType();
if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
return converter.decodeFloat(columnPage.getByte(rowId));
} else if (dataType == DataTypes.SHORT) {
return converter.decodeFloat(columnPage.getShort(rowId));
} else if (dataType == DataTypes.SHORT_INT) {
return converter.decodeFloat(columnPage.getShortInt(rowId));
} else if (dataType == DataTypes.INT) {
return converter.decodeFloat(columnPage.getInt(rowId));
} else if (dataType == DataTypes.FLOAT) {
return columnPage.getFloat(rowId);
} else if (dataType == DataTypes.BINARY) {
return converter.decodeFloat(columnPage.getByte(rowId));
} else {
throw new RuntimeException("internal error: " + this);
}
}

@Override
Expand Down
5 changes: 5 additions & 0 deletions ...c/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,11 @@ private byte[] getValueAsBytes(Object value, DataType dataType) {
b.putLong((long) value);
b.flip();
return b.array();
} else if (dataType == DataTypes.FLOAT) {
b = ByteBuffer.allocate(8);
b.putFloat((float) value);
b.flip();
return b.array();
} else if (dataType == DataTypes.DOUBLE) {
b = ByteBuffer.allocate(8);
b.putDouble((double) value);
Expand Down
7 changes: 7 additions & 0 deletions .../main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,8 @@ private static DataType fitMinMax(DataType dataType, Object max, Object min) {
return fitLongMinMax((int) max, (int) min);
} else if ((dataType == DataTypes.LONG) || (dataType == DataTypes.TIMESTAMP)) {
return fitLongMinMax((long) max, (long) min);
} else if (dataType == DataTypes.FLOAT) {
return fitLongMinMax((long) (float) max, (long) (float) min);
} else if (dataType == DataTypes.DOUBLE) {
return fitLongMinMax((long) (double) max, (long) (double) min);
} else {
Expand Down Expand Up @@ -212,6 +214,8 @@ private static DataType fitDelta(DataType dataType, Object max, Object min) {
value = (long) (short) max - (long) (short) min;
} else if (dataType == DataTypes.INT) {
value = (long) (int) max - (long) (int) min;
} else if (dataType == DataTypes.FLOAT) {
value = (long) (float) max - (long) (float) min;
} else if (dataType == DataTypes.LONG || dataType == DataTypes.TIMESTAMP) {
value = (long) max - (long) min;
// The subtraction overflowed iff the operands have opposing signs
Expand Down Expand Up @@ -329,6 +333,9 @@ private static ColumnPageCodec getColumnPageCodec(SimpleStatsResult stats,
// If absMaxValue exceeds LONG.MAX_VALUE, then go for direct compression
if ((Math.pow(10, decimalCount) * absMaxValue) > Long.MAX_VALUE) {
return new DirectCompressCodec(DataTypes.DOUBLE);
} else if (srcDataType == DataTypes.FLOAT &&
(Math.pow(10, decimalCount) * absMaxValue) > Integer.MAX_VALUE) {
return new DirectCompressCodec(DataTypes.FLOAT);
} else {
long max = (long) (Math.pow(10, decimalCount) * absMaxValue);
DataType adaptiveDataType = fitLongMinMax(max, 0);
Expand Down
28 changes: 23 additions & 5 deletions ...g/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
public class AdaptiveDeltaFloatingCodec extends AdaptiveCodec {

private Double factor;
private float floatFactor;
private long max;

public static ColumnPageCodec newInstance(DataType srcDataType, DataType targetDataType,
Expand All @@ -65,6 +66,7 @@ public AdaptiveDeltaFloatingCodec(DataType srcDataType, DataType targetDataType,
SimpleStatsResult stats, boolean isInvertedIndex) {
super(srcDataType, targetDataType, stats, isInvertedIndex);
this.factor = Math.pow(10, stats.getDecimalCount());
floatFactor = factor.floatValue();
if (srcDataType == DataTypes.FLOAT) {
this.max =
(long) ((long) Math.pow(10, stats.getDecimalCount()) * ((float) stats.getMax()));
Expand Down Expand Up @@ -185,13 +187,15 @@ public void encode(int rowId, long value) {
@Override
public void encode(int rowId, float value) {
if (targetDataType.equals(DataTypes.BYTE)) {
encodedPage.putByte(rowId, (byte) (max - (value * factor)));
encodedPage.putByte(rowId, (byte) (max - (value * floatFactor)));
} else if (targetDataType.equals(DataTypes.SHORT)) {
encodedPage.putShort(rowId, (short) (max - (value * factor)));
encodedPage.putShort(rowId, (short) (max - (value * floatFactor)));
} else if (targetDataType.equals(DataTypes.SHORT_INT)) {
encodedPage.putShortInt(rowId, (int) (max - (value * factor)));
encodedPage.putShortInt(rowId, (int) (max - (value * floatFactor)));
} else if (targetDataType.equals(DataTypes.INT)) {
encodedPage.putInt(rowId, (int) (max - (value * factor)));
encodedPage.putInt(rowId, (int) (max - (value * floatFactor)));
} else if (targetDataType.equals(DataTypes.FLOAT)) {
encodedPage.putFloat(rowId, value);
} else {
throw new RuntimeException("internal error: " + debugInfo());
}
Expand Down Expand Up @@ -229,6 +233,21 @@ public long decodeLong(int value) {
throw new RuntimeException("internal error: " + debugInfo());
}

@Override
public float decodeFloat(byte value) {
return (max - value) / floatFactor;
}

@Override
public float decodeFloat(short value) {
return (max - value) / floatFactor;
}

@Override
public float decodeFloat(int value) {
return (max - value) / floatFactor;
}

@Override
public double decodeDouble(byte value) {
return (max - value) / factor;
Expand Down Expand Up @@ -265,7 +284,6 @@ public void decodeAndFillVector(byte[] pageData, ColumnVectorInfo vectorInfo, Bi
int intSizeInBytes = DataTypes.INT.getSizeInBytes();
int longSizeInBytes = DataTypes.LONG.getSizeInBytes();
if (vectorDataType == DataTypes.FLOAT) {
float floatFactor = factor.floatValue();
if (pageDataType == DataTypes.BOOLEAN || pageDataType == DataTypes.BYTE) {
for (int i = 0; i < pageSize; i++) {
vector.putFloat(i, (max - pageData[i]) / floatFactor);
Expand Down
19 changes: 17 additions & 2 deletions ...g/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ public AdaptiveDeltaIntegralCodec(DataType srcDataType, DataType targetDataType,
this.max = (int) stats.getMax();
} else if (srcDataType == DataTypes.LONG || srcDataType == DataTypes.TIMESTAMP) {
this.max = (long) stats.getMax();
} else if (srcDataType == DataTypes.FLOAT) {
this.max = (int) (float) stats.getMax();
} else if (srcDataType == DataTypes.DOUBLE) {
this.max = (long) (double) stats.getMax();
} else if (DataTypes.isDecimal(srcDataType)) {
Expand Down Expand Up @@ -236,8 +238,6 @@ public void encode(int rowId, float value) {
encodedPage.putShortInt(rowId, (int) (max - value));
} else if (targetDataType == DataTypes.INT) {
encodedPage.putInt(rowId, (int) (max - value));
} else if (targetDataType == DataTypes.LONG) {
encodedPage.putLong(rowId, (long) (max - value));
} else {
throw new RuntimeException("internal error");
}
Expand Down Expand Up @@ -275,6 +275,21 @@ public long decodeLong(int value) {
return max - value;
}

@Override
public float decodeFloat(byte value) {
return max - value;
}

@Override
public float decodeFloat(short value) {
return max - value;
}

@Override
public float decodeFloat(int value) {
return max - value;
}

@Override
public double decodeDouble(byte value) {
return max - value;
Expand Down
29 changes: 28 additions & 1 deletion ...va/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,19 @@ public void encode(int rowId, long value) {

@Override
public void encode(int rowId, float value) {
encode(rowId, (double) value);
if (targetDataType == DataTypes.BYTE) {
encodedPage.putByte(rowId, (byte) Math.round(value * floatFactor));
} else if (targetDataType == DataTypes.SHORT) {
encodedPage.putShort(rowId, (short) Math.round(value * floatFactor));
} else if (targetDataType == DataTypes.SHORT_INT) {
encodedPage.putShortInt(rowId, Math.round(value * floatFactor));
} else if (targetDataType == DataTypes.INT) {
encodedPage.putInt(rowId, Math.round(value * floatFactor));
} else if (targetDataType == DataTypes.FLOAT) {
encodedPage.putFloat(rowId, value);
} else {
throw new RuntimeException("internal error: " + debugInfo());
}
}

@Override
Expand Down Expand Up @@ -209,6 +221,21 @@ public long decodeLong(int value) {
throw new RuntimeException("internal error: " + debugInfo());
}

@Override
public float decodeFloat(byte value) {
return value / floatFactor;
}

@Override
public float decodeFloat(short value) {
return value / floatFactor;
}

@Override
public float decodeFloat(int value) {
return value / floatFactor;
}

@Override
public double decodeDouble(byte value) {
return value / factor;
Expand Down
15 changes: 15 additions & 0 deletions ...va/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,21 @@ public long decodeLong(int value) {
return value;
}

@Override
public float decodeFloat(byte value) {
return value;
}

@Override
public float decodeFloat(short value) {
return value;
}

@Override
public float decodeFloat(int value) {
return value;
}

@Override
public double decodeDouble(byte value) {
return value;
Expand Down
15 changes: 15 additions & 0 deletions ...java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,21 @@ public long decodeLong(int value) {
return value;
}

@Override
public float decodeFloat(byte value) {
return value;
}

@Override
public float decodeFloat(short value) {
return value;
}

@Override
public float decodeFloat(int value) {
return value;
}

@Override
public double decodeDouble(byte value) {
return value;
Expand Down
Loading