From b263024ec01766f05e4f45980d75f6ca7da0db4f Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 19 Dec 2024 09:45:30 +0100 Subject: [PATCH 1/8] Add unprocessed raw data endpoint --- .../LunaticJsonRawDataUnprocessedDto.java | 11 ++++ .../rest/responses/ResponseController.java | 15 +++++- .../rawdata/LunaticJsonDataModel.java | 1 + .../ports/api/LunaticJsonRawDataApiPort.java | 6 ++- .../ports/spi/LunaticJsonPersistancePort.java | 3 ++ .../rawdata/LunaticJsonRawDataService.java | 21 +++++++- .../adapter/LunaticJsonMongoAdapter.java | 7 +++ .../rawdata/LunaticJsonDataDocument.java | 1 + .../LunaticJsonMongoDBRepository.java | 5 ++ .../responses/ResponseControllerTest.java | 50 ++++++++++++++++++- .../LunaticJsonRawDataServiceTest.java | 13 +++-- .../stubs/LunaticJsonPersistanceStub.java | 10 ++++ 12 files changed, 131 insertions(+), 12 deletions(-) create mode 100644 src/main/java/fr/insee/genesis/controller/dto/rawdata/LunaticJsonRawDataUnprocessedDto.java diff --git a/src/main/java/fr/insee/genesis/controller/dto/rawdata/LunaticJsonRawDataUnprocessedDto.java b/src/main/java/fr/insee/genesis/controller/dto/rawdata/LunaticJsonRawDataUnprocessedDto.java new file mode 100644 index 0000000..f99a1ae --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/dto/rawdata/LunaticJsonRawDataUnprocessedDto.java @@ -0,0 +1,11 @@ +package fr.insee.genesis.controller.dto.rawdata; + +import lombok.Builder; +import lombok.Data; + +@Builder +@Data +public class LunaticJsonRawDataUnprocessedDto { + private String campaignId; + private String idUE; +} diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index baac0b5..3fc020b 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -10,6 +10,7 @@ import fr.insee.genesis.controller.dto.SurveyUnitDto; import fr.insee.genesis.controller.dto.SurveyUnitId; import fr.insee.genesis.controller.dto.SurveyUnitSimplified; +import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign; import fr.insee.genesis.controller.sources.xml.LunaticXmlDataParser; import fr.insee.genesis.controller.sources.xml.LunaticXmlDataSequentialParser; @@ -155,6 +156,7 @@ public ResponseEntity saveResponsesFromXmlCampaignFolder(@RequestParam(" return ResponseEntity.internalServerError().body(errors.getFirst().getMessage()); } + //RAW DATA XML @Operation(summary = "Save one file of raw responses to Genesis Database, passing its path as a parameter") @PutMapping(path = "/lunatic-xml/raw/save-one") public ResponseEntity saveRawResponsesFromXmlFile(@RequestParam("pathLunaticXml") String xmlFile, @@ -192,17 +194,18 @@ public ResponseEntity saveRawResponsesFromXmlCampaignFolder(@RequestPara return ResponseEntity.ok(getSuccessMessage(isAnyDataSaved)); } - //JSON + //RAW DATA JSON @Operation(summary = "Save lunatic json data to Genesis Database from the campaign root folder") @PutMapping(path = "/lunatic-json/raw/save") public ResponseEntity saveRawResponsesFromJsonBody( @RequestParam("campaignName") String campaignName, + @RequestParam("idUE") String idUE, @RequestParam(value = "mode", required = false) Mode modeSpecified, @RequestBody String dataJson ) { log.info("Try to import raw lunatic JSON data for campaign: {}", campaignName); try { - lunaticJsonRawDataApiPort.saveData(campaignName, dataJson, modeSpecified); + lunaticJsonRawDataApiPort.saveData(campaignName, idUE, dataJson, modeSpecified); }catch (JsonProcessingException jpe){ log.error(jpe.toString()); return ResponseEntity.badRequest().body("Invalid JSON synthax"); @@ -211,6 +214,14 @@ public ResponseEntity saveRawResponsesFromJsonBody( return ResponseEntity.ok(SUCCESS_MESSAGE); } + //GET unprocessed + @Operation(summary = "Get campaign id and idUE from all unprocessed raw json data") + @GetMapping(path = "/lunatic-json/raw/get/unprocessed") + public ResponseEntity> getUnproccessedJsonRawData(){ + log.info("Try to get unprocessed raw JSON datas..."); + return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedData()); + } + //SAVE ALL @Operation(summary = "Save all files to Genesis Database (differential data folder only), regardless of the campaign") diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java index d7faa03..4f669da 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java @@ -11,6 +11,7 @@ public record LunaticJsonDataModel( ObjectId id, String campaignId, + String idUE, Mode mode, String dataJson, LocalDateTime recordDate, diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java index 1180e34..24bd75c 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java @@ -1,8 +1,12 @@ package fr.insee.genesis.domain.ports.api; import com.fasterxml.jackson.core.JsonProcessingException; +import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.domain.model.surveyunit.Mode; +import java.util.List; + public interface LunaticJsonRawDataApiPort { - void saveData(String campaignName, String dataJson, Mode mode) throws JsonProcessingException; + void saveData(String campaignName,String idUE, String dataJson, Mode mode) throws JsonProcessingException; + List getUnprocessedData(); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java index 5885ff3..ece63e7 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java @@ -2,6 +2,9 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; +import java.util.List; + public interface LunaticJsonPersistancePort { void save(LunaticJsonDataModel lunaticJsonDataModel); + List getAllUnprocessedData(); } diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index febdf9c..04ac337 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; @@ -13,6 +14,8 @@ import org.springframework.stereotype.Service; import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; @Service public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { @@ -25,12 +28,13 @@ public LunaticJsonRawDataService(LunaticJsonPersistancePort lunaticJsonPersistan } @Override - public void saveData(String campaignName, String dataJson, Mode mode) throws JsonParseException { + public void saveData(String campaignName, String idUE, String dataJson, Mode mode) throws JsonParseException { if(!isJsonValid(dataJson)){ throw new JsonParseException("Invalid JSON synthax"); } LunaticJsonDataModel lunaticJsonDataModel = LunaticJsonDataModel.builder() .campaignId(campaignName) + .idUE(idUE) .mode(mode) .dataJson(dataJson) .recordDate(LocalDateTime.now()) @@ -39,6 +43,21 @@ public void saveData(String campaignName, String dataJson, Mode mode) throws Jso lunaticJsonPersistancePort.save(lunaticJsonDataModel); } + @Override + public List getUnprocessedData() { + List dtos = new ArrayList<>(); + + for(LunaticJsonDataModel dataModel : lunaticJsonPersistancePort.getAllUnprocessedData()){ + dtos.add(LunaticJsonRawDataUnprocessedDto.builder() + .campaignId(dataModel.campaignId()) + .idUE(dataModel.idUE()) + .build() + ); + } + + return dtos; + } + private boolean isJsonValid(String json) { ObjectMapper mapper = new ObjectMapper() diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java index 29e1f89..6fd527a 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java @@ -10,6 +10,8 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import java.util.List; + @Slf4j @Service @Qualifier("lunaticJsonMongoAdapter") @@ -28,4 +30,9 @@ public void save(LunaticJsonDataModel lunaticJsonDataModel) { .modelToDocument(lunaticJsonDataModel); lunaticJsonMongoDBRepository.insert(document); } + + @Override + public List getAllUnprocessedData() { + return LunaticJsonDocumentMapper.INSTANCE.listDocumentToListModel(lunaticJsonMongoDBRepository.findByNullProcessDate()); + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java index e063fbb..3c17779 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java @@ -19,6 +19,7 @@ public class LunaticJsonDataDocument { @Id private ObjectId id; private String campaignId; + private String idUE; private Mode mode; private Map data; private LocalDateTime recordDate; diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java index a668ad0..84d8973 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java @@ -2,8 +2,13 @@ import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; import org.springframework.stereotype.Repository; +import java.util.List; + @Repository public interface LunaticJsonMongoDBRepository extends MongoRepository { + @Query("{\"processDate\" : null}") + List findByNullProcessDate(); } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 77dbdea..e592826 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -5,6 +5,7 @@ import fr.insee.genesis.controller.dto.SurveyUnitDto; import fr.insee.genesis.controller.dto.SurveyUnitId; import fr.insee.genesis.controller.dto.SurveyUnitSimplified; +import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.utils.ControllerUtils; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; @@ -16,6 +17,7 @@ import fr.insee.genesis.domain.service.rawdata.LunaticXmlRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.LunaticJsonPersistanceStub; @@ -165,17 +167,23 @@ void saveXmlRawDataFromFolderTest() throws Exception { //json @Test void saveJsonRawDataFromStringTest(){ + //GIVEN lunaticJsonPersistanceStub.getMongoStub().clear(); String campaignId = "SAMPLETEST-PARADATA-v1"; + String idUE = "testIdUE"; + //WHEN responseControllerStatic.saveRawResponsesFromJsonBody( campaignId + , idUE , Mode.WEB , "{\"testdata\": \"test\"}" ); + //THEN Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub()).isNotEmpty(); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getCampaignId()).isNotNull().isEqualTo(campaignId); + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getIdUE()).isNotNull().isEqualTo(idUE); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getData().get("testdata")).isNotNull(); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getData().get("testdata")).isNotNull().isEqualTo("test"); @@ -185,6 +193,38 @@ void saveJsonRawDataFromStringTest(){ } + @Test + void getUnprocessedDataTest(){ + //GIVEN + lunaticJsonPersistanceStub.getMongoStub().clear(); + String campaignId = "SAMPLETEST1"; + String idUE = "testIdUE1"; + addJsonRawDataDocumentToStub(campaignId, idUE, null); + + //WHEN + List dtos = responseControllerStatic.getUnproccessedJsonRawData().getBody(); + + //THEN + Assertions.assertThat(dtos).isNotNull().isNotEmpty().hasSize(1); + Assertions.assertThat(dtos.getFirst().getCampaignId()).isEqualTo(campaignId); + Assertions.assertThat(dtos.getFirst().getIdUE()).isEqualTo(idUE); + } + + @Test + void getUnprocessedDataTest_processDate_present(){ + //GIVEN + lunaticJsonPersistanceStub.getMongoStub().clear(); + String campaignId = "SAMPLETEST2"; + String idUE = "testIdUE2"; + addJsonRawDataDocumentToStub(campaignId, idUE, LocalDateTime.now()); + + //WHEN + List dtos = responseControllerStatic.getUnproccessedJsonRawData().getBody(); + + //THEN + Assertions.assertThat(dtos).isNotNull().isEmpty(); + } + //All data @Test void saveResponsesFromAllCampaignFoldersTests(){ @@ -376,6 +416,14 @@ void getLatestByStatesSurveyDataTest(){ .isTrue(); } - + //Utils + private static void addJsonRawDataDocumentToStub(String campaignId, String idUE, LocalDateTime processDate) { + LunaticJsonDataDocument lunaticJsonDataDocument = new LunaticJsonDataDocument(); + lunaticJsonDataDocument.setCampaignId(campaignId); + lunaticJsonDataDocument.setIdUE(idUE); + lunaticJsonDataDocument.setRecordDate(LocalDateTime.now()); + lunaticJsonDataDocument.setProcessDate(processDate); + lunaticJsonPersistanceStub.getMongoStub().add(lunaticJsonDataDocument); + } } diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index 742f451..1db8c21 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -15,12 +15,11 @@ void saveDataTest_Invalid(){ lunaticJsonPersistanceStub.getMongoStub().clear(); String campaignId = "SAMPLETEST-PARADATA-v1"; - Assertions.assertThatThrownBy(() -> { - lunaticJsonRawDataService.saveData( - campaignId - ,"{\"testdata\": \"ERROR" - , Mode.WEB - ); - }).isInstanceOf(JsonParseException.class); + Assertions.assertThatThrownBy(() -> lunaticJsonRawDataService.saveData( + campaignId + ,"TESTIDUE" + ,"{\"testdata\": \"ERROR" + , Mode.WEB + )).isInstanceOf(JsonParseException.class); } } \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java index 56c32ee..368cbd6 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java @@ -17,4 +17,14 @@ public class LunaticJsonPersistanceStub implements LunaticJsonPersistancePort { public void save(LunaticJsonDataModel lunaticJsonDataModel) { mongoStub.add(LunaticJsonDocumentMapper.INSTANCE.modelToDocument(lunaticJsonDataModel)); } + + @Override + public List getAllUnprocessedData() { + return LunaticJsonDocumentMapper.INSTANCE.listDocumentToListModel( + mongoStub.stream().filter( + lunaticJsonDataDocument -> lunaticJsonDataDocument.getProcessDate() == null + ) + .toList() + ); + } } From ff82ef88f7effc2cda881a51fe452a823f749bfe Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Fri, 20 Dec 2024 10:06:19 +0100 Subject: [PATCH 2/8] Add data processing --- src/main/java/fr/insee/genesis/Constants.java | 2 + .../rest/responses/ResponseController.java | 51 ++++++++++++- .../model/variabletype/VariableTypeModel.java | 14 ++++ .../ports/api/LunaticJsonRawDataApiPort.java | 13 +++- .../domain/ports/api/VariableTypeApiPort.java | 7 ++ .../ports/spi/LunaticJsonPersistancePort.java | 7 ++ .../spi/VariableTypePersistancePort.java | 7 ++ .../rawdata/LunaticJsonRawDataService.java | 72 +++++++++++++++++- .../variabletype/VariableTypeService.java | 38 ++++++++++ .../adapter/LunaticJsonMongoAdapter.java | 28 ++++++- .../adapter/VariableTypeMongoAdapter.java | 33 ++++++++ .../rawdata/LunaticJsonDataDocument.java | 3 +- .../variabletype/VariableTypeDocument.java | 11 +++ .../mappers/VariableTypeDocumentMapper.java | 21 ++++++ .../LunaticJsonMongoDBRepository.java | 7 ++ .../responses/ResponseControllerTest.java | 75 ++++++++++++++++++- .../stubs/LunaticJsonPersistanceStub.java | 24 ++++++ .../stubs/VariableTypePersistanceStub.java | 20 +++++ 18 files changed, 423 insertions(+), 10 deletions(-) create mode 100644 src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java create mode 100644 src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java create mode 100644 src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java create mode 100644 src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java create mode 100644 src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java index f1711c2..a3ec0eb 100644 --- a/src/main/java/fr/insee/genesis/Constants.java +++ b/src/main/java/fr/insee/genesis/Constants.java @@ -10,11 +10,13 @@ public class Constants { "(^([0-9]|[0-2][0-9]|3[0-1])[\\-\\/]([0-9]|1[0-2]|0[1-9])[\\-\\/]([0-9]{4})$)"; public static final String FILTER_RESULT_PREFIX = "FILTER_RESULT_"; public static final String MISSING_SUFFIX = "_MISSING"; + public static final String MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME = "lunaticjsondata"; private static final String[] ENO_VARIABLES = {"COMMENT_QE","COMMENT_UE","HEURE_REMPL","MIN_REMPL"}; public static final String MONGODB_SCHEDULE_COLLECTION_NAME = "schedules"; public static final String LOOP_NAME_PREFIX = "BOUCLE"; public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses"; + public static final String MONGODB_VARIABLETYPE_COLLECTION_NAME = "variabletypes"; public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries"; public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY"; public static final String VOLUMETRY_FILE_DATE_FORMAT = "yyyy_MM_dd"; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 3fc020b..bccba86 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -23,6 +23,7 @@ import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.api.LunaticXmlRawDataApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.ports.api.VariableTypeApiPort; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; @@ -72,6 +73,7 @@ public class ResponseController { private final SurveyUnitQualityService surveyUnitQualityService; private final LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort; private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort; + private final VariableTypeApiPort variableTypeApiPort; private final FileUtils fileUtils; private final ControllerUtils controllerUtils; @@ -80,12 +82,14 @@ public ResponseController(SurveyUnitApiPort surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort, LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, + VariableTypeApiPort variableTypeApiPort, FileUtils fileUtils, ControllerUtils controllerUtils) { this.surveyUnitService = surveyUnitService; this.surveyUnitQualityService = surveyUnitQualityService; this.lunaticXmlRawDataApiPort = lunaticXmlRawDataApiPort; this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort; + this.variableTypeApiPort = variableTypeApiPort; this.fileUtils = fileUtils; this.controllerUtils = controllerUtils; } @@ -200,7 +204,7 @@ public ResponseEntity saveRawResponsesFromXmlCampaignFolder(@RequestPara public ResponseEntity saveRawResponsesFromJsonBody( @RequestParam("campaignName") String campaignName, @RequestParam("idUE") String idUE, - @RequestParam(value = "mode", required = false) Mode modeSpecified, + @RequestParam(value = "mode") Mode modeSpecified, @RequestBody String dataJson ) { log.info("Try to import raw lunatic JSON data for campaign: {}", campaignName); @@ -219,7 +223,49 @@ public ResponseEntity saveRawResponsesFromJsonBody( @GetMapping(path = "/lunatic-json/raw/get/unprocessed") public ResponseEntity> getUnproccessedJsonRawData(){ log.info("Try to get unprocessed raw JSON datas..."); - return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedData()); + return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataIds()); + } + + //PROCESS + @Operation(summary = "Process raw data of a campaign") + @PostMapping(path = "/lunatic-json/raw/process") + public ResponseEntity processJsonRawData( + @RequestParam("campaignName") String campaignName, + @RequestBody List idUEList + ) throws GenesisException { + log.info("Try to get process raw JSON datas for campaign {} and {} idUEs", campaignName, idUEList.size()); + + int dataCount = 0; + List errors = new ArrayList<>(); + + List modesList = controllerUtils.getModesList(campaignName, null); + for(Mode mode: modesList){ + //Load and save metadatas into database, throw exception if none + VariablesMap variablesMap = readMetadatas(campaignName, mode, errors, true); + if(variablesMap == null){ + throw new GenesisException(400, + "Error during metadata parsing for mode %s :%n%s" + .formatted(mode, errors.getLast().toString()) + ); + } + + //Save converted data + List surveyUnitModels = lunaticJsonRawDataApiPort.parseRawData( + campaignName, + mode, + idUEList, + variablesMap + ); + surveyUnitService.saveSurveyUnits(surveyUnitModels); + + //Update process dates + lunaticJsonRawDataApiPort.updateProcessDates(surveyUnitModels); + + //Save metadatas + variableTypeApiPort.saveMetadatas(campaignName, variablesMap); + dataCount += surveyUnitModels.size(); + } + return ResponseEntity.ok("%d document(s) processed".formatted(dataCount)); } @@ -257,7 +303,6 @@ public ResponseEntity saveResponsesFromAllCampaignFolders(){ return ResponseEntity.status(209).body("Data saved with " + errors.size() + " errors"); } - //DELETE @Operation(summary = "Delete all responses associated with a questionnaire") diff --git a/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java b/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java new file mode 100644 index 0000000..3214ae3 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java @@ -0,0 +1,14 @@ +package fr.insee.genesis.domain.model.variabletype; + +import fr.insee.bpm.metadata.model.VariableType; +import lombok.Builder; +import lombok.Data; + +import java.util.Map; + +@Data +@Builder +public class VariableTypeModel { + String campaignId; + Map variableTypeMap; +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java index 24bd75c..b778baf 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java @@ -1,12 +1,23 @@ package fr.insee.genesis.domain.ports.api; import com.fasterxml.jackson.core.JsonProcessingException; +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import java.util.List; public interface LunaticJsonRawDataApiPort { void saveData(String campaignName,String idUE, String dataJson, Mode mode) throws JsonProcessingException; - List getUnprocessedData(); + List getUnprocessedDataIds(); + + List parseRawData( + String campaignName, + Mode mode, + List idUEList, + VariablesMap variablesMap + ); + + void updateProcessDates(List surveyUnitModels); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java new file mode 100644 index 0000000..99846e8 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java @@ -0,0 +1,7 @@ +package fr.insee.genesis.domain.ports.api; + +import fr.insee.bpm.metadata.model.VariablesMap; + +public interface VariableTypeApiPort { + void saveMetadatas(String campaignId, VariablesMap variablesMap); +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java index ece63e7..4c2684c 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java @@ -1,10 +1,17 @@ package fr.insee.genesis.domain.ports.spi; +import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; +import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; import java.util.List; +import java.util.Set; public interface LunaticJsonPersistancePort { void save(LunaticJsonDataModel lunaticJsonDataModel); List getAllUnprocessedData(); + + List findRawData(String campaignName, Mode mode, List idUEList); + + void updateProcessDates(String campaignId, Set idUEs); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java new file mode 100644 index 0000000..9b48a85 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java @@ -0,0 +1,7 @@ +package fr.insee.genesis.domain.ports.spi; + +import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; + +public interface VariableTypePersistancePort { + void save(VariableTypeModel variableTypeModel); +} diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 04ac337..2054fc5 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -4,18 +4,26 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; +import fr.insee.genesis.domain.model.surveyunit.CollectedVariable; +import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.spi.LunaticJsonPersistancePort; +import fr.insee.genesis.domain.utils.LoopIdentifier; +import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; @Service public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { @@ -44,7 +52,7 @@ public void saveData(String campaignName, String idUE, String dataJson, Mode mod } @Override - public List getUnprocessedData() { + public List getUnprocessedDataIds() { List dtos = new ArrayList<>(); for(LunaticJsonDataModel dataModel : lunaticJsonPersistancePort.getAllUnprocessedData()){ @@ -58,6 +66,68 @@ public List getUnprocessedData() { return dtos; } + @Override + public List parseRawData( + String campaignName, + Mode mode, + List idUEList, + VariablesMap variablesMap + ) { + //Get concerned raw data + List rawDataList = lunaticJsonPersistancePort.findRawData(campaignName, mode, idUEList); + + //Convert to genesis model + List surveyUnitModels = new ArrayList<>(); + for(LunaticJsonDataDocument rawData : rawDataList){ + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .idCampaign(campaignName) + .idQuest(campaignName) + .mode(rawData.getMode()) + .idUE(rawData.getIdUE()) + .state(DataState.COLLECTED) + .fileDate(rawData.getRecordDate()) + .recordDate(LocalDateTime.now()) + .collectedVariables(new ArrayList<>()) + .build(); + + //Data variables convertion + for(String variableName : rawData.getData().keySet()){ + CollectedVariable collectedVariable = CollectedVariable.collectedVariableBuilder() + .idVar(variableName) + .values(new ArrayList<>()) + .idLoop(variablesMap.getVariable(variableName).getGroupName()) + .idParent(LoopIdentifier.getRelatedVariableName(variableName, variablesMap)) + .build(); + + collectedVariable.getValues().add(rawData.getData().get(variableName).toString()); + + surveyUnitModel.getCollectedVariables().add(collectedVariable); + } + surveyUnitModels.add(surveyUnitModel); + } + + return surveyUnitModels; + } + + @Override + public void updateProcessDates(List surveyUnitModels) { + Set campaignIds = new HashSet<>(); + for(SurveyUnitModel surveyUnitModel : surveyUnitModels){ + campaignIds.add(surveyUnitModel.getIdCampaign()); + } + + for(String campaignId : campaignIds){ + Set idUEs = new HashSet<>(); + for(SurveyUnitModel surveyUnitModel : + surveyUnitModels.stream().filter( + surveyUnitModel -> surveyUnitModel.getIdCampaign().equals(campaignId) + ).toList()){ + idUEs.add(surveyUnitModel.getIdUE()); + } + lunaticJsonPersistancePort.updateProcessDates(campaignId, idUEs); + } + } + private boolean isJsonValid(String json) { ObjectMapper mapper = new ObjectMapper() diff --git a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java new file mode 100644 index 0000000..b297ebc --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java @@ -0,0 +1,38 @@ +package fr.insee.genesis.domain.service.variabletype; + +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; +import fr.insee.genesis.domain.ports.api.VariableTypeApiPort; +import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.util.LinkedHashMap; + +@Service +public class VariableTypeService implements VariableTypeApiPort { + @Qualifier("variableTypeMongoAdapter") + private final VariableTypePersistancePort variableTypePersistancePort; + + @Autowired + public VariableTypeService(VariableTypePersistancePort variableTypePersistancePort) { + this.variableTypePersistancePort = variableTypePersistancePort; + } + + @Override + public void saveMetadatas(String campaignId, VariablesMap variablesMap) { + VariableTypeModel variableTypeModel = VariableTypeModel.builder() + .campaignId(campaignId) + .variableTypeMap(new LinkedHashMap<>()) + .build(); + + for(String variableName : variablesMap.getVariables().keySet()){ + Variable bpmVariable = variablesMap.getVariable(variableName); + variableTypeModel.getVariableTypeMap().put(variableName,bpmVariable.getType()); + } + + variableTypePersistancePort.save(variableTypeModel); + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java index 6fd527a..0b9a27c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java @@ -1,5 +1,7 @@ package fr.insee.genesis.infrastructure.adapter; +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; import fr.insee.genesis.domain.ports.spi.LunaticJsonPersistancePort; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; @@ -8,9 +10,15 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; import org.springframework.stereotype.Service; +import java.time.LocalDateTime; import java.util.List; +import java.util.Set; @Slf4j @Service @@ -18,11 +26,13 @@ public class LunaticJsonMongoAdapter implements LunaticJsonPersistancePort { private final LunaticJsonMongoDBRepository lunaticJsonMongoDBRepository; + private final MongoTemplate mongoTemplate; @Autowired - public LunaticJsonMongoAdapter(LunaticJsonMongoDBRepository lunaticJsonMongoDBRepository) { + public LunaticJsonMongoAdapter(LunaticJsonMongoDBRepository lunaticJsonMongoDBRepository, MongoTemplate mongoTemplate) { this.lunaticJsonMongoDBRepository = lunaticJsonMongoDBRepository; - } + this.mongoTemplate = mongoTemplate; + } @Override public void save(LunaticJsonDataModel lunaticJsonDataModel) { @@ -35,4 +45,18 @@ public void save(LunaticJsonDataModel lunaticJsonDataModel) { public List getAllUnprocessedData() { return LunaticJsonDocumentMapper.INSTANCE.listDocumentToListModel(lunaticJsonMongoDBRepository.findByNullProcessDate()); } + + @Override + public List findRawData(String campaignName, Mode mode, List idUEList) { + return lunaticJsonMongoDBRepository.findModesByCampaignIdAndByModeAndIdUEInIdUEList(campaignName, mode, idUEList); + } + + @Override + public void updateProcessDates(String campaignId, Set idUEs) { + mongoTemplate.updateMulti( + Query.query(Criteria.where("campaignId").is(campaignId).and("idUE").in(idUEs)) + , new Update().set("processDate", LocalDateTime.now()) + , Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME + ); + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java new file mode 100644 index 0000000..95ef09e --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java @@ -0,0 +1,33 @@ +package fr.insee.genesis.infrastructure.adapter; + +import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; +import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; +import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; +import fr.insee.genesis.infrastructure.mappers.VariableTypeDocumentMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.FindAndReplaceOptions; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.stereotype.Service; + +@Service +@Qualifier("variableTypeMongoAdapter") +public class VariableTypeMongoAdapter implements VariableTypePersistancePort { + private final MongoTemplate mongoTemplate; + + @Autowired + public VariableTypeMongoAdapter(MongoTemplate mongoTemplate) { + this.mongoTemplate = mongoTemplate; + } + + @Override + public void save(VariableTypeModel variableTypeModel) { + mongoTemplate.update(VariableTypeDocument.class) + .matching(Query.query(Criteria.where("campaignId").is(variableTypeModel.getCampaignId()))) + .replaceWith(VariableTypeDocumentMapper.INSTANCE.modelToDocument(variableTypeModel)) + .withOptions(FindAndReplaceOptions.options().upsert()) + .findAndReplace(); + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java index 3c17779..dedd873 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.document.rawdata; +import fr.insee.genesis.Constants; import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.AllArgsConstructor; import lombok.Data; @@ -14,7 +15,7 @@ @Data @NoArgsConstructor @AllArgsConstructor -@Document(collection = "lunaticjsondata") +@Document(collection = Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME) public class LunaticJsonDataDocument { @Id private ObjectId id; diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java new file mode 100644 index 0000000..dfc1459 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java @@ -0,0 +1,11 @@ +package fr.insee.genesis.infrastructure.document.variabletype; + +import fr.insee.genesis.Constants; +import lombok.Data; +import org.springframework.data.mongodb.core.mapping.Document; + +@Data +@Document(collection = Constants.MONGODB_VARIABLETYPE_COLLECTION_NAME) +public class VariableTypeDocument { + +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java new file mode 100644 index 0000000..f3c55a8 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java @@ -0,0 +1,21 @@ +package fr.insee.genesis.infrastructure.mappers; + +import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; +import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; + +import java.util.List; + +@Mapper +public interface VariableTypeDocumentMapper { + VariableTypeDocumentMapper INSTANCE = Mappers.getMapper(VariableTypeDocumentMapper.class); + + VariableTypeModel documentToModel(VariableTypeDocument variableTypeDocument); + + VariableTypeDocument modelToDocument(VariableTypeModel variableTypeModel); + + List listDocumentToListModel(List variableTypeDocuments); + + List listModelToListDocument(List variableTypeModels); +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java index 84d8973..07054b1 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.repository; +import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.data.mongodb.repository.Query; @@ -11,4 +12,10 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository { @Query("{\"processDate\" : null}") List findByNullProcessDate(); + + @Query(value = "{ 'campaignId' : ?0 }", fields = "{ 'mode' : 1 }") + List findModesByCampaignId(String campaignId); + + @Query(value = "{ 'campaignId' : ?0, 'mode' : ?1, 'idUE': {$in: ?2} }") + List findModesByCampaignIdAndByModeAndIdUEInIdUEList(String campaignName, Mode mode, List idUEList); } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index e592826..aeadec4 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -13,16 +13,20 @@ import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.api.LunaticXmlRawDataApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.ports.api.VariableTypeApiPort; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.rawdata.LunaticXmlRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.domain.service.variabletype.VariableTypeService; +import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.LunaticJsonPersistanceStub; import fr.insee.genesis.stubs.LunaticXmlPersistanceStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; +import fr.insee.genesis.stubs.VariableTypePersistanceStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -37,6 +41,7 @@ import java.time.LocalDateTime; import java.time.Month; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; class ResponseControllerTest { @@ -45,6 +50,7 @@ class ResponseControllerTest { static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub; static LunaticXmlPersistanceStub lunaticXmlPersistanceStub; static LunaticJsonPersistanceStub lunaticJsonPersistanceStub; + static VariableTypePersistanceStub variableTypePersistanceStub; static List surveyUnitIdList; //Constants @@ -62,12 +68,16 @@ static void init() { lunaticJsonPersistanceStub = new LunaticJsonPersistanceStub(); LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonPersistanceStub); + variableTypePersistanceStub = new VariableTypePersistanceStub(); + VariableTypeApiPort variableTypeApiPort = new VariableTypeService(variableTypePersistanceStub); + FileUtils fileUtils = new FileUtils(new ConfigStub()); responseControllerStatic = new ResponseController( surveyUnitApiPort , new SurveyUnitQualityService() , lunaticXmlRawDataApiPort , lunaticJsonRawDataApiPort + , variableTypeApiPort , fileUtils , new ControllerUtils(fileUtils) ); @@ -181,7 +191,7 @@ void saveJsonRawDataFromStringTest(){ ); //THEN - Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub()).isNotEmpty(); + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub()).isNotEmpty().hasSize(1); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getCampaignId()).isNotNull().isEqualTo(campaignId); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getIdUE()).isNotNull().isEqualTo(idUE); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getMode()).isEqualTo(Mode.WEB); @@ -190,7 +200,6 @@ void saveJsonRawDataFromStringTest(){ Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getRecordDate()).isNotNull(); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getProcessDate()).isNull(); - } @Test @@ -225,6 +234,54 @@ void getUnprocessedDataTest_processDate_present(){ Assertions.assertThat(dtos).isNotNull().isEmpty(); } + //raw data process + //json + @Test + void processJsonRawDataTest() throws GenesisException { + //GIVEN + lunaticJsonPersistanceStub.getMongoStub().clear(); + surveyUnitPersistencePortStub.getMongoStub().clear(); + String campaignId = "SAMPLETEST-PARADATA-v2"; + String idUE = "testIdUE1"; + String varName = "AVIS_MAIL"; + String varValue = "TEST"; + addJsonRawDataDocumentToStub(campaignId, idUE, null, varName, varValue); + //TODO comment faire avec les boucles ? + + List idUEList = new ArrayList<>(); + idUEList.add(idUE); + + //WHEN + responseControllerStatic.processJsonRawData(campaignId, idUEList); + + + //THEN + //Genesis model survey unit created successfully + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotNull().isNotEmpty().hasSize(1); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst()).isNotNull(); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdCampaign()).isEqualTo(campaignId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdQuest()).isNotNull().isEqualTo(campaignId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getMode()).isNotNull().isEqualTo(Mode.WEB); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdUE()).isEqualTo(idUE); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getFileDate()).isNotNull(); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getRecordDate()).isNotNull(); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables()).isNotNull().isNotEmpty().hasSize(1); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables().getFirst()).isNotNull(); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables().getFirst().getIdVar()).isNotNull().isEqualTo(varName); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables().getFirst().getValues()).isNotNull().isNotEmpty().hasSize(1); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables().getFirst().getValues().getFirst()).isNotNull().isEqualTo(varValue); + + // + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getProcessDate()).isNotNull(); + + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub()).isNotEmpty().hasSize(1); + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getCampaignId()).isNotNull().isEqualTo(campaignId); + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getIdUE()).isNotNull().isEqualTo(idUE); + + //Var/type + Assertions.assertThat(variableTypePersistanceStub.getMongoStub()).isNotNull().isNotEmpty().hasSize(1); + } + //All data @Test void saveResponsesFromAllCampaignFoldersTests(){ @@ -420,9 +477,23 @@ void getLatestByStatesSurveyDataTest(){ private static void addJsonRawDataDocumentToStub(String campaignId, String idUE, LocalDateTime processDate) { LunaticJsonDataDocument lunaticJsonDataDocument = new LunaticJsonDataDocument(); lunaticJsonDataDocument.setCampaignId(campaignId); + lunaticJsonDataDocument.setMode(Mode.WEB); + lunaticJsonDataDocument.setIdUE(idUE); + lunaticJsonDataDocument.setRecordDate(LocalDateTime.now()); + lunaticJsonDataDocument.setProcessDate(processDate); + lunaticJsonPersistanceStub.getMongoStub().add(lunaticJsonDataDocument); + } + + private static void addJsonRawDataDocumentToStub(String campaignId, String idUE, LocalDateTime processDate, + String variableName, String variableValue) { + LunaticJsonDataDocument lunaticJsonDataDocument = new LunaticJsonDataDocument(); + lunaticJsonDataDocument.setCampaignId(campaignId); + lunaticJsonDataDocument.setMode(Mode.WEB); lunaticJsonDataDocument.setIdUE(idUE); lunaticJsonDataDocument.setRecordDate(LocalDateTime.now()); lunaticJsonDataDocument.setProcessDate(processDate); + lunaticJsonDataDocument.setData(new HashMap<>()); + lunaticJsonDataDocument.getData().put(variableName,variableValue); lunaticJsonPersistanceStub.getMongoStub().add(lunaticJsonDataDocument); } diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java index 368cbd6..22632cb 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java @@ -1,13 +1,16 @@ package fr.insee.genesis.stubs; +import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; import fr.insee.genesis.domain.ports.spi.LunaticJsonPersistancePort; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; import fr.insee.genesis.infrastructure.mappers.LunaticJsonDocumentMapper; import lombok.Getter; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; +import java.util.Set; @Getter public class LunaticJsonPersistanceStub implements LunaticJsonPersistancePort { @@ -27,4 +30,25 @@ public List getAllUnprocessedData() { .toList() ); } + + @Override + public List findRawData(String campaignName, Mode mode, List idUEList) { + return + mongoStub.stream().filter(lunaticJsonDataDocument -> + lunaticJsonDataDocument.getCampaignId().equals(campaignName) + && lunaticJsonDataDocument.getMode().equals(mode) + && idUEList.contains(lunaticJsonDataDocument.getIdUE()) + ).toList(); + } + + @Override + public void updateProcessDates(String campaignId, Set idUEs) { + for(LunaticJsonDataDocument document : mongoStub.stream().filter( + lunaticJsonDataDocument -> lunaticJsonDataDocument.getCampaignId().equals(campaignId) + && idUEs.contains(lunaticJsonDataDocument.getIdUE()) + ).toList() + ){ + document.setProcessDate(LocalDateTime.now()); + } + } } diff --git a/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java new file mode 100644 index 0000000..86480c8 --- /dev/null +++ b/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java @@ -0,0 +1,20 @@ +package fr.insee.genesis.stubs; + +import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; +import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; +import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; +import fr.insee.genesis.infrastructure.mappers.VariableTypeDocumentMapper; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +public class VariableTypePersistanceStub implements VariableTypePersistancePort { + List mongoStub = new ArrayList<>(); + + @Override + public void save(VariableTypeModel variableTypeModel) { + mongoStub.add(VariableTypeDocumentMapper.INSTANCE.modelToDocument(variableTypeModel)); + } +} From e44151b1ab1d453047e9383ff4f14ae4d45c0446 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Fri, 20 Dec 2024 11:40:37 +0100 Subject: [PATCH 3/8] Add variable type storage --- .../insee/genesis/configuration/Config.java | 2 ++ .../rest/responses/ResponseController.java | 8 +++-- .../rawdata/LunaticJsonDataModel.java | 1 + .../model/variabletype/VariableTypeModel.java | 7 +++-- .../ports/api/LunaticJsonRawDataApiPort.java | 2 +- .../domain/ports/api/VariableTypeApiPort.java | 3 +- .../spi/VariableTypePersistancePort.java | 5 ++++ .../rawdata/LunaticJsonRawDataService.java | 5 ++-- .../variabletype/VariableTypeService.java | 10 +++++-- .../adapter/VariableTypeMongoAdapter.java | 11 ++++++- .../rawdata/LunaticJsonDataDocument.java | 1 + .../variabletype/VariableTypeDocument.java | 9 +++++- .../mappers/VariableTypeDocumentMapper.java | 2 +- .../VariableTypeMongoDBRepository.java | 15 ++++++++++ .../responses/ResponseControllerTest.java | 29 ++++++++++++------- .../LunaticJsonRawDataServiceTest.java | 1 + .../stubs/VariableTypePersistanceStub.java | 10 +++++++ 17 files changed, 95 insertions(+), 26 deletions(-) create mode 100644 src/main/java/fr/insee/genesis/infrastructure/repository/VariableTypeMongoDBRepository.java diff --git a/src/main/java/fr/insee/genesis/configuration/Config.java b/src/main/java/fr/insee/genesis/configuration/Config.java index ed5812e..1391ff2 100644 --- a/src/main/java/fr/insee/genesis/configuration/Config.java +++ b/src/main/java/fr/insee/genesis/configuration/Config.java @@ -2,12 +2,14 @@ import lombok.Getter; import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.annotation.Configuration; import java.nio.file.Path; @Configuration @Getter +@EnableCaching public class Config { /******************************************************/ diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index bccba86..24f81d3 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -200,16 +200,17 @@ public ResponseEntity saveRawResponsesFromXmlCampaignFolder(@RequestPara //RAW DATA JSON @Operation(summary = "Save lunatic json data to Genesis Database from the campaign root folder") - @PutMapping(path = "/lunatic-json/raw/save") + @PutMapping(path = "/lunatic-json/raw/save-one") public ResponseEntity saveRawResponsesFromJsonBody( @RequestParam("campaignName") String campaignName, + @RequestParam("idQuest") String idQuest, @RequestParam("idUE") String idUE, @RequestParam(value = "mode") Mode modeSpecified, @RequestBody String dataJson ) { log.info("Try to import raw lunatic JSON data for campaign: {}", campaignName); try { - lunaticJsonRawDataApiPort.saveData(campaignName, idUE, dataJson, modeSpecified); + lunaticJsonRawDataApiPort.saveData(campaignName, idQuest, idUE, dataJson, modeSpecified); }catch (JsonProcessingException jpe){ log.error(jpe.toString()); return ResponseEntity.badRequest().body("Invalid JSON synthax"); @@ -231,6 +232,7 @@ public ResponseEntity> getUnproccessedJso @PostMapping(path = "/lunatic-json/raw/process") public ResponseEntity processJsonRawData( @RequestParam("campaignName") String campaignName, + @RequestParam("questionnaireId") String questionnaireId, @RequestBody List idUEList ) throws GenesisException { log.info("Try to get process raw JSON datas for campaign {} and {} idUEs", campaignName, idUEList.size()); @@ -262,7 +264,7 @@ public ResponseEntity processJsonRawData( lunaticJsonRawDataApiPort.updateProcessDates(surveyUnitModels); //Save metadatas - variableTypeApiPort.saveMetadatas(campaignName, variablesMap); + variableTypeApiPort.saveMetadatas(campaignName, questionnaireId, mode, variablesMap); dataCount += surveyUnitModels.size(); } return ResponseEntity.ok("%d document(s) processed".formatted(dataCount)); diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java index 4f669da..beeaed5 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java @@ -11,6 +11,7 @@ public record LunaticJsonDataModel( ObjectId id, String campaignId, + String idQuest, String idUE, Mode mode, String dataJson, diff --git a/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java b/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java index 3214ae3..6ac4dac 100644 --- a/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java @@ -1,6 +1,7 @@ package fr.insee.genesis.domain.model.variabletype; import fr.insee.bpm.metadata.model.VariableType; +import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.Builder; import lombok.Data; @@ -9,6 +10,8 @@ @Data @Builder public class VariableTypeModel { - String campaignId; - Map variableTypeMap; + private String campaignId; + private String questionnaireId; + private Mode mode; + private Map variables; } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java index b778baf..2cfb9d5 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java @@ -9,7 +9,7 @@ import java.util.List; public interface LunaticJsonRawDataApiPort { - void saveData(String campaignName,String idUE, String dataJson, Mode mode) throws JsonProcessingException; + void saveData(String campaignName, String idQuest, String idUE, String dataJson, Mode mode) throws JsonProcessingException; List getUnprocessedDataIds(); List parseRawData( diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java index 99846e8..17220bf 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java @@ -1,7 +1,8 @@ package fr.insee.genesis.domain.ports.api; import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.domain.model.surveyunit.Mode; public interface VariableTypeApiPort { - void saveMetadatas(String campaignId, VariablesMap variablesMap); + void saveMetadatas(String campaignId, String questionnaireId, Mode mode, VariablesMap variablesMap); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java index 9b48a85..3c2e363 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java @@ -1,7 +1,12 @@ package fr.insee.genesis.domain.ports.spi; +import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; +import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; +import org.springframework.cache.annotation.Cacheable; public interface VariableTypePersistancePort { void save(VariableTypeModel variableTypeModel); + @Cacheable("variableTypes") + VariableTypeDocument find(String campaignId, String questionnaireId, Mode mode); } diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 2054fc5..f325560 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -36,12 +36,13 @@ public LunaticJsonRawDataService(LunaticJsonPersistancePort lunaticJsonPersistan } @Override - public void saveData(String campaignName, String idUE, String dataJson, Mode mode) throws JsonParseException { + public void saveData(String campaignName, String idQuest, String idUE, String dataJson, Mode mode) throws JsonParseException { if(!isJsonValid(dataJson)){ throw new JsonParseException("Invalid JSON synthax"); } LunaticJsonDataModel lunaticJsonDataModel = LunaticJsonDataModel.builder() .campaignId(campaignName) + .idQuest(idQuest) .idUE(idUE) .mode(mode) .dataJson(dataJson) @@ -81,7 +82,7 @@ public List parseRawData( for(LunaticJsonDataDocument rawData : rawDataList){ SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .idCampaign(campaignName) - .idQuest(campaignName) + .idQuest(rawData.getIdQuest()) .mode(rawData.getMode()) .idUE(rawData.getIdUE()) .state(DataState.COLLECTED) diff --git a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java index b297ebc..c58871f 100644 --- a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java +++ b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java @@ -2,6 +2,7 @@ import fr.insee.bpm.metadata.model.Variable; import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; import fr.insee.genesis.domain.ports.api.VariableTypeApiPort; import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; @@ -9,6 +10,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import java.util.HashMap; import java.util.LinkedHashMap; @Service @@ -22,15 +24,17 @@ public VariableTypeService(VariableTypePersistancePort variableTypePersistancePo } @Override - public void saveMetadatas(String campaignId, VariablesMap variablesMap) { + public void saveMetadatas(String campaignId, String questionnaireId, Mode mode, VariablesMap variablesMap) { VariableTypeModel variableTypeModel = VariableTypeModel.builder() .campaignId(campaignId) - .variableTypeMap(new LinkedHashMap<>()) + .questionnaireId(questionnaireId) + .mode(mode) + .variables(new LinkedHashMap<>()) .build(); for(String variableName : variablesMap.getVariables().keySet()){ Variable bpmVariable = variablesMap.getVariable(variableName); - variableTypeModel.getVariableTypeMap().put(variableName,bpmVariable.getType()); + variableTypeModel.getVariables().put(variableName,bpmVariable.getType()); } variableTypePersistancePort.save(variableTypeModel); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java index 95ef09e..ff329e1 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java @@ -1,9 +1,11 @@ package fr.insee.genesis.infrastructure.adapter; +import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; import fr.insee.genesis.infrastructure.mappers.VariableTypeDocumentMapper; +import fr.insee.genesis.infrastructure.repository.VariableTypeMongoDBRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.FindAndReplaceOptions; @@ -15,10 +17,12 @@ @Service @Qualifier("variableTypeMongoAdapter") public class VariableTypeMongoAdapter implements VariableTypePersistancePort { + private final VariableTypeMongoDBRepository variableTypeMongoDBRepository; private final MongoTemplate mongoTemplate; @Autowired - public VariableTypeMongoAdapter(MongoTemplate mongoTemplate) { + public VariableTypeMongoAdapter(VariableTypeMongoDBRepository variableTypeMongoDBRepository, MongoTemplate mongoTemplate) { + this.variableTypeMongoDBRepository = variableTypeMongoDBRepository; this.mongoTemplate = mongoTemplate; } @@ -30,4 +34,9 @@ public void save(VariableTypeModel variableTypeModel) { .withOptions(FindAndReplaceOptions.options().upsert()) .findAndReplace(); } + + @Override + public VariableTypeDocument find(String campaignId, String questionnaireId, Mode mode) { + return variableTypeMongoDBRepository.findFirstByCampaignIdQuestionnaireIdMode(campaignId, questionnaireId, mode); + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java index dedd873..fd5aa2b 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java @@ -20,6 +20,7 @@ public class LunaticJsonDataDocument { @Id private ObjectId id; private String campaignId; + private String idQuest; private String idUE; private Mode mode; private Map data; diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java index dfc1459..61e4037 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java @@ -1,11 +1,18 @@ package fr.insee.genesis.infrastructure.document.variabletype; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.Data; import org.springframework.data.mongodb.core.mapping.Document; +import java.util.Map; + @Data @Document(collection = Constants.MONGODB_VARIABLETYPE_COLLECTION_NAME) public class VariableTypeDocument { - + private String campaignId; + private String questionnaireId; + private Mode mode; + private Map variables; } diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java index f3c55a8..ed8cdec 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java @@ -7,7 +7,7 @@ import java.util.List; -@Mapper +@Mapper(componentModel = "spring") public interface VariableTypeDocumentMapper { VariableTypeDocumentMapper INSTANCE = Mappers.getMapper(VariableTypeDocumentMapper.class); diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/VariableTypeMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/VariableTypeMongoDBRepository.java new file mode 100644 index 0000000..ac6141c --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/VariableTypeMongoDBRepository.java @@ -0,0 +1,15 @@ +package fr.insee.genesis.infrastructure.repository; + +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.stereotype.Repository; + +@Repository +public interface VariableTypeMongoDBRepository extends MongoRepository { + + @Query(value = "{ 'campaignId' : ?0 , 'questionnaireId' : ?1 , 'mode' : ?2 }") + VariableTypeDocument findFirstByCampaignIdQuestionnaireIdMode(String campaignId, String questionnaireId, Mode mode); +} diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index aeadec4..8a4fff6 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -1,6 +1,7 @@ package fr.insee.genesis.controller.rest.responses; import cucumber.TestConstants; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.genesis.Constants; import fr.insee.genesis.controller.dto.SurveyUnitDto; import fr.insee.genesis.controller.dto.SurveyUnitId; @@ -180,11 +181,13 @@ void saveJsonRawDataFromStringTest(){ //GIVEN lunaticJsonPersistanceStub.getMongoStub().clear(); String campaignId = "SAMPLETEST-PARADATA-v1"; + String idQuest = "testIdQuest"; String idUE = "testIdUE"; //WHEN responseControllerStatic.saveRawResponsesFromJsonBody( campaignId + , idQuest , idUE , Mode.WEB , "{\"testdata\": \"test\"}" @@ -193,6 +196,7 @@ void saveJsonRawDataFromStringTest(){ //THEN Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub()).isNotEmpty().hasSize(1); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getCampaignId()).isNotNull().isEqualTo(campaignId); + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getIdQuest()).isNotNull().isEqualTo(idQuest); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getIdUE()).isNotNull().isEqualTo(idUE); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getData().get("testdata")).isNotNull(); @@ -242,17 +246,17 @@ void processJsonRawDataTest() throws GenesisException { lunaticJsonPersistanceStub.getMongoStub().clear(); surveyUnitPersistencePortStub.getMongoStub().clear(); String campaignId = "SAMPLETEST-PARADATA-v2"; + String idQuest = campaignId + "_quest"; String idUE = "testIdUE1"; String varName = "AVIS_MAIL"; String varValue = "TEST"; - addJsonRawDataDocumentToStub(campaignId, idUE, null, varName, varValue); - //TODO comment faire avec les boucles ? + addJsonRawDataDocumentToStub(campaignId, idQuest, idUE, null, varName, varValue); List idUEList = new ArrayList<>(); idUEList.add(idUE); //WHEN - responseControllerStatic.processJsonRawData(campaignId, idUEList); + responseControllerStatic.processJsonRawData(campaignId, idQuest, idUEList); //THEN @@ -260,7 +264,7 @@ void processJsonRawDataTest() throws GenesisException { Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotNull().isNotEmpty().hasSize(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdCampaign()).isEqualTo(campaignId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdQuest()).isNotNull().isEqualTo(campaignId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdQuest()).isNotNull().isEqualTo(idQuest); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getMode()).isNotNull().isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdUE()).isEqualTo(idUE); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getFileDate()).isNotNull(); @@ -271,15 +275,16 @@ void processJsonRawDataTest() throws GenesisException { Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables().getFirst().getValues()).isNotNull().isNotEmpty().hasSize(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables().getFirst().getValues().getFirst()).isNotNull().isEqualTo(varValue); - // + //Process date check Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getProcessDate()).isNotNull(); - Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub()).isNotEmpty().hasSize(1); - Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getCampaignId()).isNotNull().isEqualTo(campaignId); - Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getIdUE()).isNotNull().isEqualTo(idUE); - - //Var/type + //Var/type check Assertions.assertThat(variableTypePersistanceStub.getMongoStub()).isNotNull().isNotEmpty().hasSize(1); + Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getCampaignId()).isEqualTo(campaignId); + Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getQuestionnaireId()).isEqualTo(idQuest); + Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getMode()).isEqualTo(Mode.WEB); + Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getVariables()).isNotNull().isNotEmpty().containsKey(varName); + Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getVariables().get(varName)).isEqualTo(VariableType.STRING); } //All data @@ -484,10 +489,12 @@ private static void addJsonRawDataDocumentToStub(String campaignId, String idUE, lunaticJsonPersistanceStub.getMongoStub().add(lunaticJsonDataDocument); } - private static void addJsonRawDataDocumentToStub(String campaignId, String idUE, LocalDateTime processDate, + private static void addJsonRawDataDocumentToStub(String campaignId, String idQuest, String idUE, + LocalDateTime processDate, String variableName, String variableValue) { LunaticJsonDataDocument lunaticJsonDataDocument = new LunaticJsonDataDocument(); lunaticJsonDataDocument.setCampaignId(campaignId); + lunaticJsonDataDocument.setIdQuest(idQuest); lunaticJsonDataDocument.setMode(Mode.WEB); lunaticJsonDataDocument.setIdUE(idUE); lunaticJsonDataDocument.setRecordDate(LocalDateTime.now()); diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index 1db8c21..d14acf8 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -17,6 +17,7 @@ void saveDataTest_Invalid(){ Assertions.assertThatThrownBy(() -> lunaticJsonRawDataService.saveData( campaignId + ,"TESTIDQUEST" ,"TESTIDUE" ,"{\"testdata\": \"ERROR" , Mode.WEB diff --git a/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java index 86480c8..e15220b 100644 --- a/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java @@ -1,5 +1,6 @@ package fr.insee.genesis.stubs; +import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; @@ -17,4 +18,13 @@ public class VariableTypePersistanceStub implements VariableTypePersistancePort public void save(VariableTypeModel variableTypeModel) { mongoStub.add(VariableTypeDocumentMapper.INSTANCE.modelToDocument(variableTypeModel)); } + + @Override + public VariableTypeDocument find(String campaignId, String questionnaireId, Mode mode) { + return mongoStub.stream().filter(variableTypeDocument -> + variableTypeDocument.getCampaignId().equals(campaignId) + && variableTypeDocument.getQuestionnaireId().equals(questionnaireId) + && variableTypeDocument.getMode().equals(mode) + ).toList().getFirst(); + } } From c07fdea26ef7310baeacbf97d71be956b11d0c56 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Fri, 20 Dec 2024 13:30:33 +0100 Subject: [PATCH 4/8] Add exception management + fix after mongo tests --- .../rest/responses/ResponseController.java | 56 ++++++++++--------- .../controller/utils/ControllerUtils.java | 2 +- .../rawdata/LunaticJsonRawDataService.java | 5 +- .../variabletype/VariableTypeService.java | 1 - 4 files changed, 35 insertions(+), 29 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 24f81d3..5155178 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -234,40 +234,44 @@ public ResponseEntity processJsonRawData( @RequestParam("campaignName") String campaignName, @RequestParam("questionnaireId") String questionnaireId, @RequestBody List idUEList - ) throws GenesisException { + ){ log.info("Try to get process raw JSON datas for campaign {} and {} idUEs", campaignName, idUEList.size()); int dataCount = 0; List errors = new ArrayList<>(); - List modesList = controllerUtils.getModesList(campaignName, null); - for(Mode mode: modesList){ - //Load and save metadatas into database, throw exception if none - VariablesMap variablesMap = readMetadatas(campaignName, mode, errors, true); - if(variablesMap == null){ - throw new GenesisException(400, - "Error during metadata parsing for mode %s :%n%s" - .formatted(mode, errors.getLast().toString()) + try { + List modesList = controllerUtils.getModesList(campaignName, null); + for (Mode mode : modesList) { + //Load and save metadatas into database, throw exception if none + VariablesMap variablesMap = readMetadatas(campaignName, mode, errors, true); + if (variablesMap == null) { + throw new GenesisException(400, + "Error during metadata parsing for mode %s :%n%s" + .formatted(mode, errors.getLast().getMessage()) + ); + } + + //Save converted data + List surveyUnitModels = lunaticJsonRawDataApiPort.parseRawData( + campaignName, + mode, + idUEList, + variablesMap ); - } + surveyUnitService.saveSurveyUnits(surveyUnitModels); - //Save converted data - List surveyUnitModels = lunaticJsonRawDataApiPort.parseRawData( - campaignName, - mode, - idUEList, - variablesMap - ); - surveyUnitService.saveSurveyUnits(surveyUnitModels); - - //Update process dates - lunaticJsonRawDataApiPort.updateProcessDates(surveyUnitModels); - - //Save metadatas - variableTypeApiPort.saveMetadatas(campaignName, questionnaireId, mode, variablesMap); - dataCount += surveyUnitModels.size(); + //Update process dates + lunaticJsonRawDataApiPort.updateProcessDates(surveyUnitModels); + + //Save metadatas + variableTypeApiPort.saveMetadatas(campaignName, questionnaireId, mode, variablesMap); + dataCount += surveyUnitModels.size(); + } + return ResponseEntity.ok("%d document(s) processed".formatted(dataCount)); + }catch (GenesisException e){ //TODO replace with spring exception handler + return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } - return ResponseEntity.ok("%d document(s) processed".formatted(dataCount)); } diff --git a/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java b/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java index 4ee6806..06a6a89 100644 --- a/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java +++ b/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java @@ -24,7 +24,7 @@ public ControllerUtils(FileUtils fileUtils) { public List getModesList(String campaign, Mode modeSpecified) throws GenesisException { // If a mode is specified, we treat only this mode. // If no mode is specified, we treat all modes in the campaign. - // If no node is specified and no specs are found, we return an error + // If no mode is specified and no specs are found, we return an error if (modeSpecified != null){ return Collections.singletonList(modeSpecified); } diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index f325560..7410769 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.domain.model.surveyunit.CollectedVariable; import fr.insee.genesis.domain.model.surveyunit.DataState; @@ -96,7 +97,9 @@ public List parseRawData( CollectedVariable collectedVariable = CollectedVariable.collectedVariableBuilder() .idVar(variableName) .values(new ArrayList<>()) - .idLoop(variablesMap.getVariable(variableName).getGroupName()) + .idLoop(variablesMap.getVariable(variableName) == null ? + Constants.ROOT_GROUP_NAME : //TODO What do we do if null ? Exception ? + variablesMap.getVariable(variableName).getGroupName()) .idParent(LoopIdentifier.getRelatedVariableName(variableName, variablesMap)) .build(); diff --git a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java index c58871f..13c9a8b 100644 --- a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java +++ b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java @@ -10,7 +10,6 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import java.util.HashMap; import java.util.LinkedHashMap; @Service From 265347cad1c818b0a409a9ff727ac886e1884c6b Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Mon, 30 Dec 2024 10:44:24 +0100 Subject: [PATCH 5/8] refactors --- .../rawdata/LunaticJsonRawDataUnprocessedDto.java | 7 +------ .../rest/responses/ResponseController.java | 2 +- .../genesis/controller/utils/ControllerUtils.java | 13 ++++++++++--- .../model/variabletype/VariableTypeModel.java | 14 ++++++-------- .../service/rawdata/LunaticJsonRawDataService.java | 14 +++++++++++--- .../service/variabletype/VariableTypeService.java | 2 +- .../adapter/VariableTypeMongoAdapter.java | 2 +- .../document/rawdata/LunaticJsonDataDocument.java | 6 +----- .../rest/responses/ResponseControllerTest.java | 7 +++---- 9 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/dto/rawdata/LunaticJsonRawDataUnprocessedDto.java b/src/main/java/fr/insee/genesis/controller/dto/rawdata/LunaticJsonRawDataUnprocessedDto.java index f99a1ae..b949288 100644 --- a/src/main/java/fr/insee/genesis/controller/dto/rawdata/LunaticJsonRawDataUnprocessedDto.java +++ b/src/main/java/fr/insee/genesis/controller/dto/rawdata/LunaticJsonRawDataUnprocessedDto.java @@ -1,11 +1,6 @@ package fr.insee.genesis.controller.dto.rawdata; import lombok.Builder; -import lombok.Data; @Builder -@Data -public class LunaticJsonRawDataUnprocessedDto { - private String campaignId; - private String idUE; -} +public record LunaticJsonRawDataUnprocessedDto(String campaignId, String idUE){} \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 5155178..20ce843 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -235,7 +235,7 @@ public ResponseEntity processJsonRawData( @RequestParam("questionnaireId") String questionnaireId, @RequestBody List idUEList ){ - log.info("Try to get process raw JSON datas for campaign {} and {} idUEs", campaignName, idUEList.size()); + log.info("Try to process raw JSON datas for campaign {} and {} idUEs", campaignName, idUEList.size()); int dataCount = 0; List errors = new ArrayList<>(); diff --git a/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java b/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java index 06a6a89..39f25f0 100644 --- a/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java +++ b/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java @@ -21,10 +21,17 @@ public ControllerUtils(FileUtils fileUtils) { this.fileUtils = fileUtils; } + + /** + * If a mode is specified, we treat only this mode. + * If no mode is specified, we treat all modes in the campaign. + * If no mode is specified and no specs are found, we return an error + * @param campaign campaign id to get modes + * @param modeSpecified a Mode to use, null if we want all modes available + * @return a list with the mode in modeSpecified or all modes if null + * @throws GenesisException if error in specs structure + */ public List getModesList(String campaign, Mode modeSpecified) throws GenesisException { - // If a mode is specified, we treat only this mode. - // If no mode is specified, we treat all modes in the campaign. - // If no mode is specified and no specs are found, we return an error if (modeSpecified != null){ return Collections.singletonList(modeSpecified); } diff --git a/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java b/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java index 6ac4dac..043f7b6 100644 --- a/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java @@ -3,15 +3,13 @@ import fr.insee.bpm.metadata.model.VariableType; import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.Builder; -import lombok.Data; import java.util.Map; -@Data @Builder -public class VariableTypeModel { - private String campaignId; - private String questionnaireId; - private Mode mode; - private Map variables; -} +public record VariableTypeModel ( + String campaignId, + String questionnaireId, + Mode mode, + Map variables +){} diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 7410769..db57a5d 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -16,6 +16,7 @@ import fr.insee.genesis.domain.ports.spi.LunaticJsonPersistancePort; import fr.insee.genesis.domain.utils.LoopIdentifier; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; @@ -27,6 +28,7 @@ import java.util.Set; @Service +@Slf4j public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { @Qualifier("lunaticJsonMongoAdapter") private final LunaticJsonPersistancePort lunaticJsonPersistancePort; @@ -97,9 +99,7 @@ public List parseRawData( CollectedVariable collectedVariable = CollectedVariable.collectedVariableBuilder() .idVar(variableName) .values(new ArrayList<>()) - .idLoop(variablesMap.getVariable(variableName) == null ? - Constants.ROOT_GROUP_NAME : //TODO What do we do if null ? Exception ? - variablesMap.getVariable(variableName).getGroupName()) + .idLoop(getIdLoop(variablesMap, variableName)) .idParent(LoopIdentifier.getRelatedVariableName(variableName, variablesMap)) .build(); @@ -113,6 +113,14 @@ public List parseRawData( return surveyUnitModels; } + private static String getIdLoop(VariablesMap variablesMap, String variableName) { + if (variablesMap.getVariable(variableName) == null){ + log.warn("Variable {} not present in metadatas, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); + return Constants.ROOT_GROUP_NAME; + } + return variablesMap.getVariable(variableName).getGroupName(); + } + @Override public void updateProcessDates(List surveyUnitModels) { Set campaignIds = new HashSet<>(); diff --git a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java index 13c9a8b..a9449b2 100644 --- a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java +++ b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java @@ -33,7 +33,7 @@ public void saveMetadatas(String campaignId, String questionnaireId, Mode mode, for(String variableName : variablesMap.getVariables().keySet()){ Variable bpmVariable = variablesMap.getVariable(variableName); - variableTypeModel.getVariables().put(variableName,bpmVariable.getType()); + variableTypeModel.variables().put(variableName,bpmVariable.getType()); } variableTypePersistancePort.save(variableTypeModel); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java index ff329e1..55ac040 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java @@ -29,7 +29,7 @@ public VariableTypeMongoAdapter(VariableTypeMongoDBRepository variableTypeMongoD @Override public void save(VariableTypeModel variableTypeModel) { mongoTemplate.update(VariableTypeDocument.class) - .matching(Query.query(Criteria.where("campaignId").is(variableTypeModel.getCampaignId()))) + .matching(Query.query(Criteria.where("campaignId").is(variableTypeModel.campaignId()))) .replaceWith(VariableTypeDocumentMapper.INSTANCE.modelToDocument(variableTypeModel)) .withOptions(FindAndReplaceOptions.options().upsert()) .findAndReplace(); diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java index fd5aa2b..d71a763 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java @@ -2,9 +2,7 @@ import fr.insee.genesis.Constants; import fr.insee.genesis.domain.model.surveyunit.Mode; -import lombok.AllArgsConstructor; import lombok.Data; -import lombok.NoArgsConstructor; import org.bson.types.ObjectId; import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.mapping.Document; @@ -13,10 +11,8 @@ import java.util.Map; @Data -@NoArgsConstructor -@AllArgsConstructor @Document(collection = Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME) -public class LunaticJsonDataDocument { +public class LunaticJsonDataDocument { //TODO try to use record @Id private ObjectId id; private String campaignId; diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 8a4fff6..9917023 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -20,7 +20,6 @@ import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.service.variabletype.VariableTypeService; -import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; @@ -219,8 +218,8 @@ void getUnprocessedDataTest(){ //THEN Assertions.assertThat(dtos).isNotNull().isNotEmpty().hasSize(1); - Assertions.assertThat(dtos.getFirst().getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(dtos.getFirst().getIdUE()).isEqualTo(idUE); + Assertions.assertThat(dtos.getFirst().campaignId()).isEqualTo(campaignId); + Assertions.assertThat(dtos.getFirst().idUE()).isEqualTo(idUE); } @Test @@ -241,7 +240,7 @@ void getUnprocessedDataTest_processDate_present(){ //raw data process //json @Test - void processJsonRawDataTest() throws GenesisException { + void processJsonRawDataTest() { //GIVEN lunaticJsonPersistanceStub.getMongoStub().clear(); surveyUnitPersistencePortStub.getMongoStub().clear(); From 7a57e42bbec2708dfffcb8fd515e7b8eb63e43bd Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Mon, 6 Jan 2025 08:25:14 +0100 Subject: [PATCH 6/8] Removed unused --- .../genesis/domain/ports/spi/LunaticXmlPersistancePort.java | 2 -- .../insee/genesis/domain/ports/spi/SchedulePersistencePort.java | 2 +- .../genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticXmlPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticXmlPersistancePort.java index 1fc6a11..13b2003 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticXmlPersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticXmlPersistancePort.java @@ -2,8 +2,6 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; -import java.util.List; - public interface LunaticXmlPersistancePort { void save(LunaticXmlDataModel lunaticXmlDataModels); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SchedulePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SchedulePersistencePort.java index 8e3f857..f226f22 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/SchedulePersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SchedulePersistencePort.java @@ -16,5 +16,5 @@ public interface SchedulePersistencePort { long countSchedules(); - public List removeExpiredSchedules(ScheduleModel scheduleModel); + List removeExpiredSchedules(ScheduleModel scheduleModel); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java index 14c0207..6ad5f4b 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java @@ -8,7 +8,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.stereotype.Service; @Slf4j From 80f2fdf71badf670ba28cf3d89bffff9b175821b Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Mon, 6 Jan 2025 15:30:20 +0100 Subject: [PATCH 7/8] Store variablesmap in variabletype collection --- .../domain/model/variabletype/VariableTypeModel.java | 6 ++---- .../domain/service/variabletype/VariableTypeService.java | 6 ++---- .../document/variabletype/VariableTypeDocument.java | 6 ++---- .../controller/rest/responses/ResponseControllerTest.java | 5 +++-- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java b/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java index 043f7b6..c0e79a4 100644 --- a/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java @@ -1,15 +1,13 @@ package fr.insee.genesis.domain.model.variabletype; -import fr.insee.bpm.metadata.model.VariableType; +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.Builder; -import java.util.Map; - @Builder public record VariableTypeModel ( String campaignId, String questionnaireId, Mode mode, - Map variables + VariablesMap variablesMap ){} diff --git a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java index a9449b2..52ed64f 100644 --- a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java +++ b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java @@ -10,8 +10,6 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import java.util.LinkedHashMap; - @Service public class VariableTypeService implements VariableTypeApiPort { @Qualifier("variableTypeMongoAdapter") @@ -28,12 +26,12 @@ public void saveMetadatas(String campaignId, String questionnaireId, Mode mode, .campaignId(campaignId) .questionnaireId(questionnaireId) .mode(mode) - .variables(new LinkedHashMap<>()) + .variablesMap(new VariablesMap()) .build(); for(String variableName : variablesMap.getVariables().keySet()){ Variable bpmVariable = variablesMap.getVariable(variableName); - variableTypeModel.variables().put(variableName,bpmVariable.getType()); + variableTypeModel.variablesMap().putVariable(bpmVariable); } variableTypePersistancePort.save(variableTypeModel); diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java index 61e4037..5fb3cc6 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java @@ -1,18 +1,16 @@ package fr.insee.genesis.infrastructure.document.variabletype; -import fr.insee.bpm.metadata.model.VariableType; +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.Constants; import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.Data; import org.springframework.data.mongodb.core.mapping.Document; -import java.util.Map; - @Data @Document(collection = Constants.MONGODB_VARIABLETYPE_COLLECTION_NAME) public class VariableTypeDocument { private String campaignId; private String questionnaireId; private Mode mode; - private Map variables; + private VariablesMap variablesMap; } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 9917023..9a8ed9d 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -282,8 +282,9 @@ void processJsonRawDataTest() { Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getCampaignId()).isEqualTo(campaignId); Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getQuestionnaireId()).isEqualTo(idQuest); Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getMode()).isEqualTo(Mode.WEB); - Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getVariables()).isNotNull().isNotEmpty().containsKey(varName); - Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getVariables().get(varName)).isEqualTo(VariableType.STRING); + Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getVariablesMap()).isNotNull(); + Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getVariablesMap().getVariables()).isNotNull().isNotEmpty().containsKey(varName); + Assertions.assertThat(variableTypePersistanceStub.getMongoStub().getFirst().getVariablesMap().getVariables().get(varName).getType()).isEqualTo(VariableType.STRING); } //All data From 0cf7a1df126de878f01a5906df3df953b96ba64d Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 9 Jan 2025 14:51:19 +0100 Subject: [PATCH 8/8] fix: cucumber definitions with vartype --- src/test/java/cucumber/functional_tests/MainDefinitions.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/cucumber/functional_tests/MainDefinitions.java b/src/test/java/cucumber/functional_tests/MainDefinitions.java index 95deef9..70f09d4 100644 --- a/src/test/java/cucumber/functional_tests/MainDefinitions.java +++ b/src/test/java/cucumber/functional_tests/MainDefinitions.java @@ -20,12 +20,14 @@ import fr.insee.genesis.domain.service.rawdata.LunaticXmlRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.domain.service.variabletype.VariableTypeService; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.LunaticJsonPersistanceStub; import fr.insee.genesis.stubs.LunaticXmlPersistanceStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; +import fr.insee.genesis.stubs.VariableTypePersistanceStub; import io.cucumber.java.After; import io.cucumber.java.Before; import io.cucumber.java.en.Given; @@ -59,6 +61,7 @@ public class MainDefinitions { surveyUnitQualityService, new LunaticXmlRawDataService(new LunaticXmlPersistanceStub()), new LunaticJsonRawDataService(new LunaticJsonPersistanceStub()), + new VariableTypeService(new VariableTypePersistanceStub()), new FileUtils(config), new ControllerUtils(new FileUtils(config)) );