From 55f3f961e2a6301cdb2fd493af373f8fd2f3059d Mon Sep 17 00:00:00 2001 From: Martin Vanek Date: Thu, 7 Mar 2024 19:08:34 +0000 Subject: [PATCH] feat: Add short, integer, long, boolean conversions into string --- .../storage/v1/JsonToProtoMessage.java | 6 +++ .../storage/v1/JsonStreamWriterTest.java | 16 ++++---- .../storage/v1/JsonToProtoMessageTest.java | 38 ++++++++++++++----- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index 6cde31081a..be785d0fad 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -650,6 +650,9 @@ private void fillField( if (val instanceof String) { protoMsg.setField(fieldDescriptor, val); return; + } else if (val instanceof Short || val instanceof Integer || val instanceof Long || val instanceof Boolean) { + protoMsg.setField(fieldDescriptor, String.valueOf(val)); + return; } break; case DOUBLE: @@ -910,6 +913,9 @@ private void fillRepeatedField( case STRING: if (val instanceof String) { protoMsg.addRepeatedField(fieldDescriptor, val); + } else if (val instanceof Short || val instanceof Integer || val instanceof Long || val instanceof Boolean) { + protoMsg.addRepeatedField(fieldDescriptor, String.valueOf(val)); + return; } else { throwWrongFieldType(fieldDescriptor, currentScope, index); } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java index d3a25510f8..a9d9b8bd3c 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java @@ -37,6 +37,7 @@ import com.google.cloud.bigquery.storage.v1.ConnectionWorkerPool.Settings; import com.google.cloud.bigquery.storage.v1.Exceptions.AppendSerializationError; import com.google.cloud.bigquery.storage.v1.TableFieldSchema.Mode; +import com.google.common.collect.ImmutableMap; import com.google.protobuf.ByteString; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Int64Value; @@ -1406,8 +1407,8 @@ public void testMultipleAppendSerializationErrors() // put a vaild value into the field foo1.put("foo", "allen"); JSONObject foo2 = new JSONObject(); - // put a number into a string field - foo2.put("foo", 666); + // put a field which is not part of the expected schema + foo2.put("not_bar", "woody"); JSONArray jsonArr = new JSONArray(); jsonArr.put(foo); jsonArr.put(foo1); @@ -1425,14 +1426,11 @@ public void testMultipleAppendSerializationErrors() } catch (AppendSerializationError appendSerializationError) { Map rowIndexToErrorMessage = appendSerializationError.getRowIndexToErrorMessage(); - assertEquals(2, rowIndexToErrorMessage.size()); - assertEquals( - "The source object has fields unknown to BigQuery: root.not_foo.", - rowIndexToErrorMessage.get(0)); assertEquals( - "Field root.foo failed to convert to STRING. Error: JSONObject does not have a string" - + " field at root.foo.", - rowIndexToErrorMessage.get(2)); + ImmutableMap.of( + 0, "The source object has fields unknown to BigQuery: root.not_foo.", + 2, "The source object has fields unknown to BigQuery: root.not_bar." + ), rowIndexToErrorMessage); } } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index dd3a6dcfa1..3f1123833b 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -87,7 +87,12 @@ public class JsonToProtoMessageTest { }) .put( StringType.getDescriptor(), - new Message[] {StringType.newBuilder().setTestFieldType("test").build()}) + new Message[] { + StringType.newBuilder().setTestFieldType("9223372036854775807").build(), + StringType.newBuilder().setTestFieldType("2147483647").build(), + StringType.newBuilder().setTestFieldType("true").build(), + StringType.newBuilder().setTestFieldType("test").build() + }) .put( RepeatedType.getDescriptor(), new Message[] { @@ -147,6 +152,9 @@ public class JsonToProtoMessageTest { .put( RepeatedString.getDescriptor(), new Message[] { + RepeatedString.newBuilder().addTestRepeated("9223372036854775807").build(), + RepeatedString.newBuilder().addTestRepeated("2147483647").build(), + RepeatedString.newBuilder().addTestRepeated("true").build(), RepeatedString.newBuilder().addTestRepeated("hello").addTestRepeated("test").build() }) .put( @@ -925,6 +933,8 @@ public void testAllTypes() throws Exception { } else if (entry.getKey() == Int64Type.getDescriptor() || entry.getKey() == BytesType.getDescriptor()) { assertEquals(entry.getKey().getFullName(), 2, success); + } else if(entry.getKey() == StringType.getDescriptor()) { + assertEquals(entry.getKey().getFullName(), 4, success); } else { assertEquals(entry.getKey().getFullName(), 1, success); } @@ -962,6 +972,8 @@ public void testAllRepeatedTypesWithLimits() throws Exception { assertEquals(entry.getKey().getFullName(), 4, success); } else if (entry.getKey() == RepeatedInt64.getDescriptor()) { assertEquals(entry.getKey().getFullName(), 2, success); + } else if (entry.getKey() == RepeatedString.getDescriptor()) { + assertEquals(entry.getKey().getFullName(), 4, success); } else { assertEquals(entry.getKey().getFullName(), 1, success); } @@ -1007,16 +1019,22 @@ public void testRequired() throws Exception { } } - @Test - public void testStructSimple() throws Exception { + @Test + public void testStructSimple() throws Exception { + structSimple("test", "test"); + structSimple(true, "true"); + structSimple(1, "1"); + structSimple((short) 1, "1"); + structSimple((long) 1, "1"); + } + + private void structSimple(Object value, String expected) throws Exception { MessageType expectedProto = MessageType.newBuilder() - .setTestFieldType(StringType.newBuilder().setTestFieldType("test").build()) + .setTestFieldType(StringType.newBuilder().setTestFieldType(expected).build()) .build(); - JSONObject stringType = new JSONObject(); - stringType.put("test_field_type", "test"); - JSONObject json = new JSONObject(); - json.put("test_field_type", stringType); + JSONObject stringType = new JSONObject(ImmutableMap.of("test_field_type", value)); + JSONObject json = new JSONObject(ImmutableMap.of("test_field_type", stringType)); DynamicMessage protoMsg = JsonToProtoMessage.INSTANCE.convertToProtoMessage(MessageType.getDescriptor(), json); @@ -1026,7 +1044,7 @@ public void testStructSimple() throws Exception { @Test public void testStructSimpleFail() throws Exception { JSONObject stringType = new JSONObject(); - stringType.put("test_field_type", 1); + stringType.put("test_field_type", new boolean[0]); JSONObject json = new JSONObject(); json.put("test_field_type", stringType); try { @@ -1268,7 +1286,7 @@ public void testNestedRepeatedComplex() throws Exception { @Test public void testNestedRepeatedComplexFail() throws Exception { double[] doubleArr = {1.1, 2.2, 3.3, 4.4, 5.5}; - Boolean[] fakeStringArr = {true, false}; + Boolean[][] fakeStringArr = {new Boolean[0], new Boolean[0]}; int[] intArr = {1, 2, 3, 4, 5}; JSONObject json = new JSONObject();