Merge branch 'master' into isolate_fhir_sync

Dockerfile

@@ -30,7 +30,7 @@ COPY ./coverage ./coverage
 
 # Updating license will fail in e2e and there is no point doing it here anyways.
 # Note this build can be faster by excluding some uber-jars we don't copy.
-RUN mvn --batch-mode clean package -Dlicense.skip=true
+RUN mvn --batch-mode clean package -Dlicense.skip=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -T 1C
 
 FROM eclipse-temurin:17-jdk-focal as main
 

bunsen/bunsen-avro/src/main/java/com/cerner/bunsen/avro/AvroConverter.java

@@ -20,7 +20,7 @@
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Field;
 import org.apache.avro.generic.IndexedRecord;
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 
 /** Converter to change HAPI objects into Avro structures and vice versa. */

bunsen/bunsen-avro/src/main/java/com/cerner/bunsen/avro/converters/DefinitionToAvroVisitor.java

@@ -140,6 +140,8 @@ public Schema getDataType() {
           .put("time", STRING_CONVERTER)
           .put("string", STRING_CONVERTER)
           .put("oid", STRING_CONVERTER)
+          // Note `xhtml` types are currently skipped because of:
+          // https://github.com/google/fhir-data-pipes/issues/1014
           .put("xhtml", STRING_CONVERTER)
           .put("decimal", DOUBLE_CONVERTER)
           .put("integer", INTEGER_CONVERTER)

bunsen/bunsen-avro/src/test/java/com/cerner/bunsen/avro/AvroConverterMergeTest.java

@@ -16,6 +16,7 @@
 import org.apache.avro.Schema.Parser;
 import org.apache.avro.generic.IndexedRecord;
 import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.r4.model.Base;
 import org.hl7.fhir.r4.model.Patient;
 import org.junit.Assert;
 import org.junit.Before;
@@ -116,9 +117,9 @@ public void validateR4UsCoreResourceWithExtension() throws ProfileException, IOE
             loadResource(fhirContext, "/r4-us-core-resources/patient_us_core.json", Patient.class);
     IndexedRecord avroRecord = patientConverter.resourceToAvro(patient);
     Patient patientDecoded = (Patient) patientConverter.avroToResource(avroRecord);
-    // For why this is needed, see: https://github.com/google/fhir-data-pipes/issues/1003
-    patientDecoded.setId(patient.getIdElement());
-    Assert.assertTrue(patient.equalsDeep(patientDecoded));
+    Assert.assertTrue(
+        patient.equalsDeep(
+            (Base) TestUtil.encodeThenParse(patientDecoded, Patient.class, fhirContext)));
   }
 
   @Test

bunsen/bunsen-avro/src/test/java/com/cerner/bunsen/avro/R4AvroConverterUsCoreTest.java

@@ -23,6 +23,7 @@
 import org.apache.avro.compiler.specific.SpecificCompiler;
 import org.apache.avro.generic.GenericData.Record;
 import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.r4.model.Base;
 import org.hl7.fhir.r4.model.CodeableConcept;
 import org.hl7.fhir.r4.model.Coding;
 import org.hl7.fhir.r4.model.Condition;
@@ -43,7 +44,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-// TODO refactor the shared code with AvroConverterTest (STU3).
+// TODO refactor the shared code with `Stu3AvroConverterUsCoreTest`.
 public class R4AvroConverterUsCoreTest {
 
   private static final Observation testObservation = TestData.newObservation();
@@ -263,8 +264,8 @@ public void testChoice() throws FHIRException {
   public void testIdInNestedElement() throws FHIRException {
 
     // Ensure that nested elements do not have id as property.
+    Assert.assertNotNull(testPatient.getAddress().get(0).getId());
     Assert.assertNull(testPatientDecoded.getAddress().get(0).getId());
-    Assert.assertNull(testPatientDecoded.getName().get(0).getId());
   }
 
   /**
@@ -587,11 +588,8 @@ public void testCompile() throws IOException {
   @Test
   public void testMetaElement() {
 
-    String id = testPatient.getId();
     Meta meta = testPatient.getMeta();
 
-    Assert.assertEquals(id, testPatientDecoded.getId());
-
     Assert.assertEquals(meta.getTag().size(), testPatientDecoded.getMeta().getTag().size());
     Assert.assertEquals(
         meta.getTag().get(0).getCode(), testPatientDecoded.getMeta().getTag().get(0).getCode());
@@ -610,4 +608,14 @@ public void testEncounterConversions() {
   public void setTestQuestionnaireResponseConversions() {
     Assert.assertTrue(testQuestionnaireResponse.equalsDeep(testQuestionnaireResponseDecoded));
   }
+
+  @Test
+  public void identicalConvertedResource() {
+    // We need to remove id of inner fields as they are intentionally removed in Avro conversion.
+    Patient tempPatient = testPatient.copy();
+    tempPatient.getAddress().get(0).setId(null);
+    Assert.assertTrue(
+        tempPatient.equalsDeep(
+            (Base) TestUtil.encodeThenParse(testPatientDecoded, Patient.class, fhirContext)));
+  }
 }

bunsen/bunsen-avro/src/test/java/com/cerner/bunsen/avro/Stu3AvroConverterUsCoreTest.java

@@ -22,6 +22,7 @@
 import org.apache.avro.Schema;
 import org.apache.avro.compiler.specific.SpecificCompiler;
 import org.apache.avro.generic.GenericData.Record;
+import org.hl7.fhir.dstu3.model.Base;
 import org.hl7.fhir.dstu3.model.CodeableConcept;
 import org.hl7.fhir.dstu3.model.Coding;
 import org.hl7.fhir.dstu3.model.Condition;
@@ -203,8 +204,8 @@ public void testChoice() throws FHIRException {
   public void testIdInNestedElement() throws FHIRException {
 
     // Ensure that nested elements do not have id as property.
+    Assert.assertNotNull(testPatient.getAddress().get(0).getId());
     Assert.assertNull(testPatientDecoded.getAddress().get(0).getId());
-    Assert.assertNull(testPatientDecoded.getName().get(0).getId());
   }
 
   /**
@@ -530,15 +531,22 @@ public void testCompile() throws IOException {
   @Test
   public void testMetaElement() {
 
-    String id = testPatient.getId();
     Meta meta = testPatient.getMeta();
 
-    Assert.assertEquals(id, testPatientDecoded.getId());
-
     Assert.assertEquals(meta.getTag().size(), testPatientDecoded.getMeta().getTag().size());
     Assert.assertEquals(
         meta.getTag().get(0).getCode(), testPatientDecoded.getMeta().getTag().get(0).getCode());
     Assert.assertEquals(
         meta.getTag().get(0).getSystem(), testPatientDecoded.getMeta().getTag().get(0).getSystem());
   }
+
+  @Test
+  public void identicalConvertedResource() {
+    // We need to remove id of inner fields as they are intentionally removed in Avro conversion.
+    Patient tempPatient = testPatient.copy();
+    tempPatient.getAddress().get(0).setId(null);
+    Assert.assertTrue(
+        tempPatient.equalsDeep(
+            (Base) TestUtil.encodeThenParse(testPatientDecoded, Patient.class, fhirContext)));
+  }
 }

bunsen/bunsen-avro/src/test/java/com/cerner/bunsen/avro/TestUtil.java

@@ -0,0 +1,30 @@
+package com.cerner.bunsen.avro;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.parser.IParser;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+
+public class TestUtil {
+
+  /**
+   * This is used when we want to verify that no information is lost in the process of converting to
+   * Avro records. The idea is to convert the resource to the JSON representation then parse it back
+   * to a HAPI object and verify we get the original HAPI object. This is needed because HAPI adds
+   * some information, like `resourceType` in `id` fields, when parsing from JSON but, we don't have
+   * that in the `id` field of converted Avro fields. See:
+   * https://github.com/google/fhir-data-pipes/issues/1003 Also see:
+   * https://github.com/google/fhir-data-pipes/issues/1214
+   *
+   * @param resource the resource which is created from an Avro record.
+   * @param resourceClass the Class of the resource.
+   * @param fhirContext the FHIR context.
+   * @return a HAPI object created from parsing the encoded version of `resource`.
+   */
+  public static IBaseResource encodeThenParse(
+      IBaseResource resource,
+      Class<? extends IBaseResource> resourceClass,
+      FhirContext fhirContext) {
+    IParser jsonParser = fhirContext.newJsonParser();
+    return jsonParser.parseResource(resourceClass, jsonParser.encodeResourceToString(resource));
+  }
+}

bunsen/bunsen-core-r4/src/test/java/com/cerner/bunsen/r4/TestData.java

@@ -5,7 +5,6 @@
 import java.util.List;
 import org.hl7.fhir.r4.model.Address;
 import org.hl7.fhir.r4.model.BooleanType;
-import org.hl7.fhir.r4.model.CodeType;
 import org.hl7.fhir.r4.model.CodeableConcept;
 import org.hl7.fhir.r4.model.Coding;
 import org.hl7.fhir.r4.model.Condition;
@@ -224,15 +223,14 @@ public static Patient newPatient() {
 
     Patient patient = new Patient();
 
-    patient.setId("test-patient");
+    patient.setId("Patient/test-patient");
     patient.setGender(AdministrativeGender.MALE);
     patient.setActive(true);
     patient.setMultipleBirth(new IntegerType(1));
 
     // This is to introduce conflict with managingOrganization identifier
     // and make sure conflict is resolved.
     Identifier patientIdentifier = new Identifier();
-    patientIdentifier.setId("patient123");
     patientIdentifier.getAssigner().setReference("Organization/123");
     patient.setIdentifier(List.of(patientIdentifier));
 
@@ -241,29 +239,28 @@ public static Patient newPatient() {
     patient.addGeneralPractitioner().setReference("Practitioner/12345");
 
     Identifier practitionerIdentifier = new Identifier();
-    practitionerIdentifier.setId("P123456");
     practitionerIdentifier.getAssigner().setReference("Organization/123456");
     patient.getGeneralPractitionerFirstRep().setIdentifier(practitionerIdentifier);
 
     Identifier managingOrganisationIdentifier = new Identifier();
-    managingOrganisationIdentifier.setId("O123456");
     managingOrganisationIdentifier.getAssigner().setReference("Organization/234");
     patient.getManagingOrganization().setIdentifier(managingOrganisationIdentifier);
 
     Address address = patient.addAddress();
     address.addLine("123 Fake Street");
+    address.setId("address123");
     address.setCity("Chicago");
     address.setState("IL");
     address.setDistrict("12345");
 
-    Extension birthSex = patient.addExtension();
-    birthSex.setUrl(US_CORE_BIRTHSEX);
-    birthSex.setValue(new CodeType("M"));
-
     Extension ethnicity = patient.addExtension();
     ethnicity.setUrl(US_CORE_ETHNICITY);
     ethnicity.setValue(null);
 
+    Extension birthSex = patient.addExtension();
+    birthSex.setUrl(US_CORE_BIRTHSEX);
+    birthSex.setValue(new StringType("M"));
+
     Coding ombCoding = new Coding();
 
     ombCoding.setSystem("urn:oid:2.16.840.1.113883.6.238");

bunsen/bunsen-core-stu3/src/test/java/com/cerner/bunsen/stu3/TestData.java

@@ -4,7 +4,6 @@
 import java.util.List;
 import org.hl7.fhir.dstu3.model.Address;
 import org.hl7.fhir.dstu3.model.BooleanType;
-import org.hl7.fhir.dstu3.model.CodeType;
 import org.hl7.fhir.dstu3.model.CodeableConcept;
 import org.hl7.fhir.dstu3.model.Coding;
 import org.hl7.fhir.dstu3.model.Condition;
@@ -185,15 +184,14 @@ public static Patient newPatient() {
 
     Patient patient = new Patient();
 
-    patient.setId("test-patient");
+    patient.setId("Patient/test-patient");
     patient.setGender(AdministrativeGender.MALE);
     patient.setActive(true);
     patient.setMultipleBirth(new IntegerType(1));
 
     // This is to introduce conflict with managingOrganization identifier
     // and make sure conflict is resolved.
     Identifier patientIdentifier = new Identifier();
-    patientIdentifier.setId("patient123");
     patientIdentifier.getAssigner().setReference("Organization/123");
     patient.setIdentifier(List.of(patientIdentifier));
 
@@ -202,12 +200,10 @@ public static Patient newPatient() {
     patient.addGeneralPractitioner().setReference("Practitioner/12345");
 
     Identifier practitionerIdentifier = new Identifier();
-    practitionerIdentifier.setId("P123456");
     practitionerIdentifier.getAssigner().setReference("Organization/123456");
     patient.getGeneralPractitionerFirstRep().setIdentifier(practitionerIdentifier);
 
     Identifier managingOrganisationIdentifier = new Identifier();
-    managingOrganisationIdentifier.setId("O123456");
     managingOrganisationIdentifier.getAssigner().setReference("Organization/234");
     patient.getManagingOrganization().setIdentifier(managingOrganisationIdentifier);
 
@@ -218,15 +214,14 @@ public static Patient newPatient() {
     address.setState("IL");
     address.setDistrict("12345");
 
-    Extension birthSex = patient.addExtension();
-
-    birthSex.setUrl(US_CORE_BIRTHSEX);
-    birthSex.setValue(new CodeType("M"));
-
     Extension ethnicity = patient.addExtension();
     ethnicity.setUrl(US_CORE_ETHNICITY);
     ethnicity.setValue(null);
 
+    Extension birthSex = patient.addExtension();
+    birthSex.setUrl(US_CORE_BIRTHSEX);
+    birthSex.setValue(new StringType("M"));
+
     Coding ombCoding = new Coding();
 
     ombCoding.setSystem("urn:oid:2.16.840.1.113883.6.238");
@@ -268,7 +263,6 @@ public static Patient newPatient() {
     humanName.setFamily("family_name");
     humanName.addGiven("given_name");
     humanName.addGiven("middle_name");
-    humanName.setId("name123");
     patient.addName(humanName);
 
     Meta meta = new Meta();

bunsen/bunsen-core/src/main/java/com/cerner/bunsen/definitions/PrimitiveConverter.java

@@ -63,7 +63,13 @@ protected Object fromHapi(IPrimitiveType primitive) {
 
   @Override
   public Object fromHapi(Object input) {
-
+    // TODO: remove this hack! It is added to address this bug:
+    // https://github.com/google/fhir-data-pipes/issues/1014
+    // with root cause being:
+    // https://github.com/hapifhir/org.hl7.fhir.core/issues/1800
+    if (!(input instanceof IPrimitiveType)) {
+      return "";
+    }
     return fromHapi((IPrimitiveType) input);
   }
 

bunsen/bunsen-core/src/main/java/com/cerner/bunsen/definitions/StructureDefinitions.java

@@ -36,6 +36,8 @@ public abstract class StructureDefinitions {
           .add("time")
           .add("string")
           .add("oid")
+          // Note `xhtml` types are currently skipped because of:
+          // https://github.com/google/fhir-data-pipes/issues/1014
           .add("xhtml")
           .add("decimal")
           .add("integer")