From 30c23dcc2dea49bd7971199d8376cac678f56041 Mon Sep 17 00:00:00 2001 From: Alexis Szmundy Date: Tue, 2 Jun 2026 11:14:37 +0200 Subject: [PATCH 01/12] Reapply "feat(responses): get responses by list of ids" This reverts commit 9ffcb6c1cde690dfd0b0cbaba94a0a0d5b6a05fa. --- .../rest/responses/ResponseController.java | 6 +- .../domain/ports/api/SurveyUnitApiPort.java | 6 +- .../ports/spi/SurveyUnitPersistencePort.java | 6 +- .../service/surveyunit/SurveyUnitService.java | 37 ++++- .../adapter/SurveyUnitMongoAdapter.java | 14 +- .../SurveyUnitMongoDBRepository.java | 18 +- .../responses/ResponseControllerTest.java | 10 +- .../surveyunit/SurveyUnitServiceTest.java | 155 +++++++++++++++++- .../adapter/SurveyUnitMongoAdapterTest.java | 136 ++++++++++++++- 9 files changed, 359 insertions(+), 29 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index eba3557f4..959d71543 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -286,7 +286,7 @@ public ResponseEntity findResponsesByInterrogationAndCollectionInstrumen @PreAuthorize("hasRole('ADMIN')") public ResponseEntity> getLatestByInterrogationAndCollectionInstrument(@RequestParam("interrogationId") String interrogationId, @RequestParam("collectionInstrumentId") String collectionInstrumentId) { - List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); + List responses = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId); return ResponseEntity.ok(responses); } @@ -300,7 +300,7 @@ public ResponseEntity> getLatestByInterrogationAndCollecti public ResponseEntity getLatestByInterrogationOneObject(@RequestParam("interrogationId") String interrogationId, @RequestParam("collectionInstrumentId") String collectionInstrumentId, @RequestParam("mode") Mode mode) { - List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); + List responses = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId); List outputVariables = new ArrayList<>(); List outputExternalVariables = new ArrayList<>(); RawResponseDto.QuestionnaireStateEnum questionnaireState = null; @@ -365,7 +365,7 @@ public ResponseEntity> getLatestForInterrogationLi List results = new ArrayList<>(); List modes = surveyUnitService.findModesByCollectionInstrumentId(collectionInstrumentId); interrogationIds.forEach(interrogationId -> { - List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List responses = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( interrogationId.getInterrogationId(), collectionInstrumentId ); modes.forEach(mode -> { diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java index e3189b015..d5e1604d0 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java @@ -26,7 +26,11 @@ public interface SurveyUnitApiPort { List findByInterrogationId(String interrogationId); - List findLatestByIdAndByCollectionInstrumentId(String interrogationId, String collectionInstrumentId); + List findLatestByInterrogationIdAndCollectionInstrumentId(String interrogationId, String collectionInstrumentId); + List findLatestByInterrogationIds( + String collectionInstrumentOrQuestionnaireId, + Set interrogationIds + ); SurveyUnitSimplifiedDto findSimplified( String collectionInstrumentId, diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java index 0eed337b6..596b1cc35 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java @@ -17,12 +17,12 @@ public interface SurveyUnitPersistencePort { List findByUsualSurveyUnitAndCollectionInstrumentIds(String usualSurveyUnitId, String collectionInstrumentId); - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ - List findBySetOfIdsAndQuestionnaireIdAndMode(String questionnaireId, String mode, List interrogationIdSet); - //========= OPTIMISATIONS PERFS (START) ========== + List findByQuestionnaireIdAndModeAndInterrogationIds(String questionnaireId, String mode, List interrogationIdSet); + + List findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds(String collectionInstrumentOrQuestionnaireId, List interrogationIdSet); List findByInterrogationId(String interrogationId); diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index fca387fac..8fb8d66b7 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -88,9 +88,32 @@ public List findByInterrogationId(String interrogationId) { * @return the latest update for each variable of a survey unit */ @Override - public List findLatestByIdAndByCollectionInstrumentId(String interrogationId, String collectionInstrumentId) { - List latestUpdatesbyVariables = new ArrayList<>(); + public List findLatestByInterrogationIdAndCollectionInstrumentId(String interrogationId, String collectionInstrumentId) { List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, collectionInstrumentId); + return getLatestSurveyUnitModels(surveyUnitModels); + } + + /** + * @param collectionInstrumentOrQuestionnaireId : Collection instrument id/questionnaire id + * @param interrogationIdSet : Interrogation ids + * @return the latest updates for given interrogationIds + */ + @Override + public List findLatestByInterrogationIds( + String collectionInstrumentOrQuestionnaireId, + Set interrogationIdSet + ) { + List surveyUnitModels = surveyUnitPersistencePort.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + collectionInstrumentOrQuestionnaireId, + interrogationIdSet.stream().toList() + ); + + return getLatestSurveyUnitModels(surveyUnitModels); + } + + private List getLatestSurveyUnitModels(List surveyUnitModels) { + List latestUpdatesByVariables = new ArrayList<>(); + List modes = getDistinctsModes(surveyUnitModels); modes.forEach(mode ->{ List suByMode = surveyUnitModels.stream() @@ -99,7 +122,7 @@ public List findLatestByIdAndByCollectionInstrumentId(String in .toList(); //We had all the variables of the oldest update - latestUpdatesbyVariables.add(suByMode.getFirst()); + latestUpdatesByVariables.add(suByMode.getFirst()); //We keep the name of already added variables to skip them in older updates Set addedVariables = new HashSet<>(); SurveyUnitModel latestUpdate = suByMode.getFirst(); @@ -152,11 +175,11 @@ public List findLatestByIdAndByCollectionInstrumentId(String in if (!collectedVariablesToKeep.isEmpty() || !externalVariablesToKeep.isEmpty()){ surveyUnitModel.setCollectedVariables(collectedVariablesToKeep); surveyUnitModel.setExternalVariables(externalVariablesToKeep); - latestUpdatesbyVariables.add(surveyUnitModel); + latestUpdatesByVariables.add(surveyUnitModel); } }); }); - return latestUpdatesbyVariables; + return latestUpdatesByVariables; } private void addDataStateIntoList(List variableModelList, DataState state){ @@ -197,7 +220,7 @@ public SurveyUnitSimplifiedDto findSimplified( String interrogationId, Mode mode, Instant recordedBefore) throws NoDataException { - List responses = findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); + List responses = findLatestByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId); if(responses.isEmpty()){ String errorMessage = "No response found for interrogation %s".formatted(interrogationId); @@ -294,7 +317,7 @@ public List> findLatestByIdAndByQuestionnaireIdAndModeOrde List queryInParam = interrogationIds.stream().map(InterrogationId::getInterrogationId).toList(); //Get !!!all versions!!! of a set of "interrogationIds" - List allResponsesVersionsSet = surveyUnitPersistencePort.findBySetOfIdsAndQuestionnaireIdAndMode(questionnaireId, mode, queryInParam); + List allResponsesVersionsSet = surveyUnitPersistencePort.findByQuestionnaireIdAndModeAndInterrogationIds(questionnaireId, mode, queryInParam); //2) FILTER BY interrogationId AND ORDER BY DATE (MOST RECENT FIRST, oldest last) interrogationIds.forEach(interrogationId -> { diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index f74b8d18a..d2becf426 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -73,12 +73,24 @@ public List findByUsualSurveyUnitAndCollectionInstrumentIds(Str * @author Adrien Marchal */ @Override - public List findBySetOfIdsAndQuestionnaireIdAndMode(String questionnaireId, String mode, List interrogationIdSet) { + public List findByQuestionnaireIdAndModeAndInterrogationIds(String questionnaireId, String mode, List interrogationIdSet) { List surveyUnits = mongoRepository.findBySetOfIdsAndQuestionnaireIdAndMode(questionnaireId, mode, interrogationIdSet); return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); } //========= OPTIMISATIONS PERFS (END) ========== + @Override + public List findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + String collectionInstrumentOrQuestionnaireId, List interrogationIds + ) { + List surveyUnits = new ArrayList<>(); + + surveyUnits.addAll(mongoRepository.findByQuestionnaireIdAndInterrogationIds(collectionInstrumentOrQuestionnaireId, interrogationIds)); + surveyUnits.addAll(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds(collectionInstrumentOrQuestionnaireId, interrogationIds)); + + return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); + } + @Override public List findByInterrogationId(String interrogationId) { List surveyUnits = mongoRepository.findByInterrogationId(interrogationId); diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java index e2e200426..47218777b 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java @@ -30,9 +30,25 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findBySetOfIdsAndQuestionnaireIdAndMode(String questionnaireId, String mode, List interrogationIdSet); + List findBySetOfIdsAndQuestionnaireIdAndMode( + String questionnaireId, + String mode, + List interrogationIds + ); //========= OPTIMISATIONS PERFS (END) ========== + @Query("{ 'questionnaireId' : ?0, 'interrogationId' : { $in: ?2 } }") + List findByQuestionnaireIdAndInterrogationIds( + String questionnaireId, + List interrogationIds + ); + @Query("{ 'collectionInstrumentId' : ?0, 'interrogationId' : { $in: ?2 } }") + List findByCollectionInstrumentIdAndInterrogationIds( + String collectionInstrumentId, + List interrogationIds + ); + + @Query(value = "{ 'questionnaireId' : ?0 }", fields = "{ 'interrogationId' : 1, 'mode' : 1 }") List findInterrogationIdsByQuestionnaireId(String questionnaireId); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index c57fd2d5a..63aa80e35 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -278,7 +278,7 @@ class GetLatestByInterrogationAndCollectionInstrumentTests { @DisplayName("Should return 200 with list of models") void getLatest_shouldReturn200() throws Exception { // GIVEN - when(surveyUnitApiPort.findLatestByIdAndByCollectionInstrumentId("INTERRO01", "QUEST01")) + when(surveyUnitApiPort.findLatestByInterrogationIdAndCollectionInstrumentId("INTERRO01", "QUEST01")) .thenReturn(List.of()); // WHEN / THEN @@ -298,7 +298,7 @@ class GetLatestByInterrogationOneObjectTests { void getLatestOneObject_shouldReturn200AndAggregateVariables() throws Exception { // GIVEN SurveyUnitModel model = buildSurveyUnitModel("INTERRO01", "QUEST01", Mode.WEB, DataState.COLLECTED); - when(surveyUnitApiPort.findLatestByIdAndByCollectionInstrumentId("INTERRO01", "QUEST01")) + when(surveyUnitApiPort.findLatestByInterrogationIdAndCollectionInstrumentId("INTERRO01", "QUEST01")) .thenReturn(List.of(model)); // WHEN / THEN @@ -316,7 +316,7 @@ void getLatestOneObject_shouldFilterByMode() throws Exception { // GIVEN SurveyUnitModel webModel = buildSurveyUnitModel("INTERRO01", "QUEST01", Mode.WEB, DataState.COLLECTED); SurveyUnitModel paperModel = buildSurveyUnitModel("INTERRO01", "QUEST01", Mode.PAPER, DataState.COLLECTED); - when(surveyUnitApiPort.findLatestByIdAndByCollectionInstrumentId("INTERRO01", "QUEST01")) + when(surveyUnitApiPort.findLatestByInterrogationIdAndCollectionInstrumentId("INTERRO01", "QUEST01")) .thenReturn(List.of(webModel, paperModel)); // WHEN / THEN @@ -374,7 +374,7 @@ void getLatestList_shouldReturn200() throws Exception { SurveyUnitModel model = buildSurveyUnitModel("INTERRO01", "QUEST01", Mode.WEB, DataState.COLLECTED); when(surveyUnitApiPort.findModesByCollectionInstrumentId("QUEST01")) .thenReturn(List.of(Mode.WEB)); - when(surveyUnitApiPort.findLatestByIdAndByCollectionInstrumentId("INTERRO01", "QUEST01")) + when(surveyUnitApiPort.findLatestByInterrogationIdAndCollectionInstrumentId("INTERRO01", "QUEST01")) .thenReturn(List.of(model)); // WHEN / THEN @@ -399,7 +399,7 @@ void getLatestList_noVariables_shouldReturnEmptyList() throws Exception { .build(); when(surveyUnitApiPort.findModesByCollectionInstrumentId("QUEST01")) .thenReturn(List.of(Mode.WEB)); - when(surveyUnitApiPort.findLatestByIdAndByCollectionInstrumentId("INTERRO01", "QUEST01")) + when(surveyUnitApiPort.findLatestByInterrogationIdAndCollectionInstrumentId("INTERRO01", "QUEST01")) .thenReturn(List.of(modelNoVars)); // WHEN / THEN diff --git a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java index 9e7e1e2c6..341f747a3 100644 --- a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java @@ -37,14 +37,19 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; class SurveyUnitServiceTest { @@ -74,7 +79,7 @@ void get_latest_should_return_usualSurveyId() { doReturn(SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(Collections.singletonList(surveyUnitDocument))) .when(surveyUnitPersistencePortStub).findByIds(any(), any()); - List surveyUnitModels = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List surveyUnitModels = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( TestConstants.DEFAULT_INTERROGATION_ID, TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -106,7 +111,7 @@ void get_latest_should_return_state_in_collected_variables(){ .when(surveyUnitPersistencePortStub).findByIds(any(), any()); //WHEN - List surveyUnitModels = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List surveyUnitModels = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( TestConstants.DEFAULT_INTERROGATION_ID, TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -140,7 +145,7 @@ void get_latest_should_return_state_in_external_variables(){ .when(surveyUnitPersistencePortStub).findByIds(any(), any()); //WHEN - List surveyUnitModels = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List surveyUnitModels = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( TestConstants.DEFAULT_INTERROGATION_ID, TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -317,7 +322,7 @@ void get_latest_should_return_usualSurveyId_when_idue() { doReturn(SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(Collections.singletonList(surveyUnitDocument))) .when(surveyUnitPersistencePortStub).findByIds(any(), any()); - List surveyUnitModels = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List surveyUnitModels = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( TestConstants.DEFAULT_INTERROGATION_ID, TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -359,7 +364,7 @@ void get_latest_should_return_edited() { doReturn(SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnitDocuments)) .when(surveyUnitPersistencePortStub).findByIds(any(), any()); - List surveyUnitModels = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List surveyUnitModels = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( TestConstants.DEFAULT_INTERROGATION_ID, TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -781,6 +786,146 @@ private VariablesMap buildVariablesMapWithVar() { } } + @Nested + @DisplayName("findLatestByInterrogationIds tests") + class findLatestByInterrogationIdsTests { + @Test + @DisplayName("Should return empty list if no survey unit is found") + void shouldReturnEmptyList() { + // GIVEN + String questionnaireId = "QUESTIONNAIRE"; + Set interrogationIds = Set.of("INT1", "INT2"); + + when(surveyUnitPersistencePortStub.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + eq(questionnaireId), + anyList())) + .thenReturn(List.of()); + + // WHEN + List result = + surveyUnitService.findLatestByInterrogationIds(questionnaireId, interrogationIds); + + // THEN + assertThat(result).isEmpty(); + + verify(surveyUnitPersistencePortStub) + .findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + eq(questionnaireId), + argThat(ids -> ids.containsAll(interrogationIds) && ids.size() == 2) + ); + } + + @Test + @DisplayName("Should return survey unit (only 1 version)") + void should_return_latest_survey_unit_when_single_version_exists() { + // GIVEN + String questionnaireId = "QUESTIONNAIRE"; + Set interrogationIds = Set.of("INT1"); + + SurveyUnitModel surveyUnit = SurveyUnitModel.builder() + .interrogationId("INT1") + .mode(Mode.WEB) + .recordDate(Instant.now()) + .collectedVariables(new ArrayList<>(List.of( + VariableModel.builder() + .varId("VAR1") + .iteration(1) + .value("VALUE1") + .build()))) + .externalVariables(new ArrayList<>()) + .build(); + + when(surveyUnitPersistencePortStub.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + eq(questionnaireId), + anyList())) + .thenReturn(List.of(surveyUnit)); + + // WHEN + List result = + surveyUnitService.findLatestByInterrogationIds(questionnaireId, interrogationIds); + + // THEN + assertThat(result) + .hasSize(1) + .first() + .extracting(SurveyUnitModel::getInterrogationId) + .isEqualTo("INT1"); + } + + @Test + @DisplayName("Should return survey unit (multiple versions)") + void should_keep_latest_variable_values_when_multiple_versions_exist() { + // GIVEN + String questionnaireId = "QUESTIONNAIRE"; + Set interrogationIds = Set.of("INT1"); + + Instant latestDate = Instant.parse("2025-01-02T10:00:00Z"); + Instant oldestDate = Instant.parse("2025-01-01T10:00:00Z"); + + VariableModel latestVariable = VariableModel.builder() + .varId("VAR1") + .iteration(1) + .value("NEW_VALUE") + .build(); + + VariableModel oldVariable = VariableModel.builder() + .varId("VAR1") + .iteration(1) + .value("OLD_VALUE") + .build(); + + VariableModel oldVariableOnlyInOldVersion = VariableModel.builder() + .varId("VAR2") + .iteration(1) + .value("VALUE2") + .build(); + + SurveyUnitModel latestVersion = SurveyUnitModel.builder() + .interrogationId("INT1") + .mode(Mode.WEB) + .recordDate(latestDate) + .collectedVariables(new ArrayList<>(List.of(latestVariable))) + .externalVariables(new ArrayList<>()) + .build(); + + SurveyUnitModel oldestVersion = SurveyUnitModel.builder() + .interrogationId("INT1") + .mode(Mode.WEB) + .recordDate(oldestDate) + .collectedVariables(new ArrayList<>(List.of(oldVariable, oldVariableOnlyInOldVersion))) + .externalVariables(new ArrayList<>()) + .build(); + + when(surveyUnitPersistencePortStub.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + eq(questionnaireId), + anyList())) + .thenReturn(List.of(oldestVersion, latestVersion)); + + // WHEN + List result = + surveyUnitService.findLatestByInterrogationIds(questionnaireId, interrogationIds); + + // THEN + assertThat(result).hasSize(2); + + SurveyUnitModel latestResult = result.get(0); + SurveyUnitModel additionalVariablesResult = result.get(1); + + assertThat(latestResult.getCollectedVariables()) + .extracting(VariableModel::varId) + .containsExactly("VAR1"); + + assertThat(latestResult.getCollectedVariables()) + .extracting(VariableModel::value) + .containsExactly("NEW_VALUE"); + + assertThat(additionalVariablesResult.getCollectedVariables()) + .extracting(VariableModel::varId) + .containsExactly("VAR2"); + } + + } + //UTILS private SurveyUnitDocument buildDoc(String interrogationId, Mode mode) { diff --git a/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapterTest.java b/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapterTest.java index d7aaf8ee4..ab5b18e4a 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapterTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapterTest.java @@ -44,7 +44,6 @@ class SurveyUnitMongoAdapterTest { private static final String QUESTIONNAIRE_ID = "questionnaire-123"; private static final String COLLECTION_INSTRUMENT_ID = "instrument-456"; private static final String INTERROGATION_ID = "interrogation-789"; - private static final String CAMPAIGN_ID = "campaign-101"; private static final String MODE = "CAWI"; @Mock @@ -180,7 +179,7 @@ void findBySetOfIds_shouldReturnMappedModels() { .thenReturn(List.of(buildDoc("i1"))); // WHEN - List result = adapter.findBySetOfIdsAndQuestionnaireIdAndMode(QUESTIONNAIRE_ID, MODE, ids); + List result = adapter.findByQuestionnaireIdAndModeAndInterrogationIds(QUESTIONNAIRE_ID, MODE, ids); // THEN assertThat(result).hasSize(1); @@ -193,13 +192,144 @@ void findBySetOfIds_noResults_shouldReturnEmptyList() { when(mongoRepository.findBySetOfIdsAndQuestionnaireIdAndMode(any(), any(), any())).thenReturn(List.of()); // WHEN - List result = adapter.findBySetOfIdsAndQuestionnaireIdAndMode(QUESTIONNAIRE_ID, MODE, List.of()); + List result = adapter.findByQuestionnaireIdAndModeAndInterrogationIds(QUESTIONNAIRE_ID, MODE, List.of()); // THEN assertThat(result).isEmpty(); } } + @Nested + @DisplayName("findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds() tests") + class FindByCollectionInstrumentIdAndModeAndInterrogationIdsTests { + @Test + @DisplayName("Should return models from both repository methods combined") + void find_shouldCombineBothRepositoryResults() { + // GIVEN + SurveyUnitDocument docByQuestionnaire = buildDoc("INTERRO01"); + SurveyUnitDocument docByCollectionInstrument = buildDoc("INTERRO02"); + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds("QUEST01", List.of("INTERRO01", "INTERRO02"))) + .thenReturn(List.of(docByQuestionnaire)); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds("QUEST01", List.of("INTERRO01", "INTERRO02"))) + .thenReturn(List.of(docByCollectionInstrument)); + + // WHEN + List result = adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01", "INTERRO02")); + + // THEN + assertThat(result).hasSize(2); + } + + @Test + @DisplayName("Should return models from questionnaireId repository only when collectionInstrument returns empty") + void find_onlyQuestionnaireResults_shouldReturnThose() { + // GIVEN + SurveyUnitDocument doc = buildDoc("INTERRO01"); + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds("QUEST01", List.of("INTERRO01"))) + .thenReturn(List.of(doc)); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds("QUEST01", List.of("INTERRO01"))) + .thenReturn(List.of()); + + // WHEN + List result = adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01")); + + // THEN + assertThat(result).hasSize(1); + } + + @Test + @DisplayName("Should return models from collectionInstrumentId repository only when questionnaire returns empty") + void find_onlyCollectionInstrumentResults_shouldReturnThose() { + // GIVEN + SurveyUnitDocument doc = buildDoc("INTERRO01"); + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds("QUEST01", List.of("INTERRO01"))) + .thenReturn(List.of()); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds("QUEST01", List.of("INTERRO01"))) + .thenReturn(List.of(doc)); + + // WHEN + List result = adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01")); + + // THEN + assertThat(result).hasSize(1); + } + + @Test + @DisplayName("Should return empty list when both repositories return empty") + void find_bothEmpty_shouldReturnEmptyList() { + // GIVEN + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + + // WHEN + List result = adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01")); + + // THEN + assertThat(result).isEmpty(); + } + + @Test + @DisplayName("Should pass the same id to both repository methods") + void find_shouldPassSameIdToBothRepositories() { + // GIVEN + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + + // WHEN + adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01")); + + // THEN + verify(mongoRepository).findByQuestionnaireIdAndInterrogationIds(eq("QUEST01"), any()); + verify(mongoRepository).findByCollectionInstrumentIdAndInterrogationIds(eq("QUEST01"), any()); + } + + @Test + @DisplayName("Should pass the same interrogationId list to both repository methods") + void find_shouldPassSameInterrogationIdListToBothRepositories() { + // GIVEN + List interrogationIds = List.of("INTERRO01", "INTERRO02", "INTERRO03"); + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + + // WHEN + adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds("QUEST01", interrogationIds); + + // THEN + verify(mongoRepository).findByQuestionnaireIdAndInterrogationIds(any(), eq(interrogationIds)); + verify(mongoRepository).findByCollectionInstrumentIdAndInterrogationIds(any(), eq(interrogationIds)); + } + + @Test + @DisplayName("Should return mapped models (not raw documents)") + void find_shouldReturnMappedModels() { + // GIVEN + SurveyUnitDocument doc = buildDoc("INTERRO01"); + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds(any(), any())) + .thenReturn(List.of(doc)); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + + // WHEN + List result = adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01")); + + // THEN + assertThat(result).hasSize(1); + assertThat(result.getFirst().getInterrogationId()).isEqualTo("INTERRO01"); + } + } + @Nested @DisplayName("findByInterrogationId() tests") class FindByInterrogationIdTests { From edfc29c4f8d6084dedf24fc33a2a45a866d99cc3 Mon Sep 17 00:00:00 2001 From: Alexis Szmundy Date: Wed, 3 Jun 2026 16:16:58 +0200 Subject: [PATCH 02/12] test: KO tests --- .../rawdata/LunaticJsonRawDataConverter.java | 37 +- .../converter/rawdata/RawDataConverter.java | 45 + ....java => RawResponseRawDataConverter.java} | 75 +- .../ports/api/LunaticJsonRawDataApiPort.java | 3 +- .../rawdata/LunaticJsonRawDataService.java | 9 +- .../service/rawdata/RawResponseService.java | 16 +- .../LunaticJsonRawDataConverterTest.java | 580 +++++++++++- .../rawdata/RawDataConverterTest.java | 89 ++ .../rawdata/RawResponseConverterTest.java | 314 ------- .../RawResponseRawDataConverterTest.java | 868 ++++++++++++++++++ .../LunaticJsonRawDataServiceTest.java | 31 +- .../rawdata/RawResponseServiceUnitTest.java | 33 +- 12 files changed, 1716 insertions(+), 384 deletions(-) create mode 100644 src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java rename src/main/java/fr/insee/genesis/domain/converter/rawdata/{RawResponseConverter.java => RawResponseRawDataConverter.java} (80%) create mode 100644 src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java delete mode 100644 src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverterTest.java create mode 100644 src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java index 66f99ff96..cd11607cf 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java @@ -9,37 +9,55 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.parser.rawdata.LunaticJsonRawDataPayloadParser; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.GroupUtils; import fr.insee.genesis.domain.utils.JsonUtils; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Component @Slf4j -@RequiredArgsConstructor -public class LunaticJsonRawDataConverter { +public class LunaticJsonRawDataConverter extends RawDataConverter{ private final LunaticJsonRawDataPayloadParser payloadParser; + public LunaticJsonRawDataConverter(SurveyUnitService surveyUnitService, LunaticJsonRawDataPayloadParser lunaticJsonRawDataPayloadParser) { + super(surveyUnitService); + this.payloadParser = lunaticJsonRawDataPayloadParser; + } + public List convertRawData( + String questionnaireId, List rawDataList, VariablesMap variablesMap ) { - return convertRawDataAndCollectEmptyModels(rawDataList, variablesMap, new ArrayList<>()); + return convertRawDataAndCollectEmptyModels( + questionnaireId, + rawDataList, + variablesMap, + new ArrayList<>() + ); } public List convertRawDataAndCollectEmptyModels( + String questionnaireId, List rawDataList, VariablesMap variablesMap, List emptySurveyUnitModels ) { List surveyUnitModels = new ArrayList<>(); + Map lastSurveyUnitModelsByInterrogationId = getLastSurveyUnitModels( + questionnaireId, + rawDataList.stream().map(LunaticJsonRawDataModel::interrogationId).collect(Collectors.toList()) + ); for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) { for (LunaticJsonRawDataModel rawData : rawDataList) { @@ -60,7 +78,14 @@ public List convertRawDataAndCollectEmptyModels( .externalVariables(new ArrayList<>()) .build(); - convertRawDataCollectedVariables(rawData, surveyUnitModel, dataState, rawDataModelType, variablesMap); + convertRawDataCollectedVariables( + rawData, + lastSurveyUnitModelsByInterrogationId.get(rawData.interrogationId()), + surveyUnitModel, + dataState, + rawDataModelType, + variablesMap + ); if (dataState == DataState.COLLECTED) { convertRawDataExternalVariables(rawData, surveyUnitModel, rawDataModelType, variablesMap); @@ -93,6 +118,7 @@ private static RawDataModelType getRawDataModelType(LunaticJsonRawDataModel rawD private void convertRawDataCollectedVariables( LunaticJsonRawDataModel srcRawData, + @Nullable SurveyUnitModel lastSurveyUnitModel, SurveyUnitModel dstSurveyUnitModel, DataState dataState, RawDataModelType rawDataModelType, @@ -116,10 +142,11 @@ private void convertRawDataCollectedVariables( List destination = dstSurveyUnitModel.getCollectedVariables(); for (Map.Entry collectedVariable : collectedMap.entrySet()) { - RawResponseConverter.processCollectedVariable( + RawResponseRawDataConverter.processCollectedVariable( collectedVariable, stateKey, variablesMap, + lastSurveyUnitModel, dstSurveyUnitModel, destination ); diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java new file mode 100644 index 000000000..d0db6951f --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java @@ -0,0 +1,45 @@ +package fr.insee.genesis.domain.converter.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Component +@NoArgsConstructor(force = true) +@RequiredArgsConstructor +public abstract class RawDataConverter { + + @Autowired + private final SurveyUnitService surveyUnitService; + + + /** + * @param questionnaireOrCollectionInstrumentId Questionnaire/Collection instrument id + * @param interrogationIds list of interrogation ids + * @return a Map containing latest survey unit models for each interrogation ids + */ + protected Map getLastSurveyUnitModels( + String questionnaireOrCollectionInstrumentId, + List interrogationIds + ) { + Set interrogationIdsSet = new HashSet<>(interrogationIds); + + return surveyUnitService.findLatestByInterrogationIds( + questionnaireOrCollectionInstrumentId, + interrogationIdsSet + ).stream().collect(Collectors.toMap( + SurveyUnitModel::getInterrogationId, + surveyUnitModel -> surveyUnitModel + )); + } +} diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java similarity index 80% rename from src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverter.java rename to src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java index ce9dec62e..59aaf91a7 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java @@ -8,46 +8,81 @@ import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.GroupUtils; import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.modelefiliere.RawResponseDto; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService.getValueString; @Component @Slf4j -@RequiredArgsConstructor -public class RawResponseConverter { +public class RawResponseRawDataConverter extends RawDataConverter { private final RawResponsePayloadParser rawResponsePayloadParser; + public RawResponseRawDataConverter(SurveyUnitService surveyUnitService, RawResponsePayloadParser rawResponsePayloadParser) { + super(surveyUnitService); + this.rawResponsePayloadParser = rawResponsePayloadParser; + } + + /** + * Like convertRawResponseAndCollectEmptyModels but ignoring the empty raw responses + */ public List convertRawResponse( + String collectionInstrumentId, List rawResponseModels, VariablesMap variablesMap ) { - return convertRawResponseAndCollectEmptyModels(rawResponseModels, variablesMap, new ArrayList<>()); + return convertRawResponseAndCollectEmptyModels( + collectionInstrumentId, + rawResponseModels, + variablesMap, + new ArrayList<>() + ); } + /** + * Converts RawResponseModels into SurveyUnitModels + * @param collectionInstrumentId Collection instrument id of raw responses to convert + * @param rawResponseModels raw responses to convert + * @param variablesMap variables map of the collection instrument + * @param emptySurveyUnitModels A list of survey units that will be filled with empty raw responses + * @return a list of SurveyUnitModels converted from raw responses + */ public List convertRawResponseAndCollectEmptyModels( + String collectionInstrumentId, List rawResponseModels, VariablesMap variablesMap, List emptySurveyUnitModels ) { List surveyUnitModels = new ArrayList<>(); + Map lastSurveyUnitModelsByInterrogationId = getLastSurveyUnitModels( + collectionInstrumentId, + rawResponseModels.stream().map(RawResponseModel::interrogationId).collect(Collectors.toList()) + ); + for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) { for (RawResponseModel rawResponseModel : rawResponseModels) { SurveyUnitModel surveyUnitModel = buildSurveyUnitModel(rawResponseModel, dataState); - convertCollectedVariables(rawResponseModel, surveyUnitModel, dataState, variablesMap); + convertCollectedVariables( + rawResponseModel, + lastSurveyUnitModelsByInterrogationId.get(rawResponseModel.interrogationId()), + surveyUnitModel, + dataState, + variablesMap + ); if (dataState == DataState.COLLECTED) { convertExternalVariables(rawResponseModel, surveyUnitModel, variablesMap); @@ -106,6 +141,7 @@ private SurveyUnitModel buildSurveyUnitModel(RawResponseModel rawResponseModel, private void convertCollectedVariables( RawResponseModel rawResponseModel, + @Nullable SurveyUnitModel lastSurveyUnitModel, SurveyUnitModel dstSurveyUnitModel, DataState dataState, VariablesMap variablesMap @@ -125,7 +161,14 @@ private void convertCollectedVariables( List collectedVariables = dstSurveyUnitModel.getCollectedVariables(); for (Map.Entry entry : collectedMap.entrySet()) { - processCollectedVariable(entry, stateKey, variablesMap, dstSurveyUnitModel, collectedVariables); + processCollectedVariable( + entry, + stateKey, + variablesMap, + lastSurveyUnitModel, + dstSurveyUnitModel, + collectedVariables + ); } } @@ -133,6 +176,7 @@ public static void processCollectedVariable( Map.Entry entry, String stateKey, VariablesMap variablesMap, + @Nullable SurveyUnitModel lastSurveyUnitModel, SurveyUnitModel dstSurveyUnitModel, List variableModelList ) { @@ -142,20 +186,25 @@ public static void processCollectedVariable( } Map states = JsonUtils.asMap(entry.getValue()); - if (states == null) { + if (states == null || states.get(stateKey) == null) { + convertNullVar(entry, lastSurveyUnitModel, variableModelList); return; } Object value = states.get(stateKey); - if (value == null) { - return; - } - if (value instanceof List list) { convertListVar(list, entry, variablesMap, variableModelList); - } else { - convertOneVar(entry, getValueString(value), variablesMap, 1, variableModelList); + return; } + convertOneVar(entry, getValueString(value), variablesMap, 1, variableModelList); + } + + private static void convertNullVar( + Map.Entry variableEntry, + @Nullable SurveyUnitModel lastSurveyUnitModel, + List destination + ) { + //TODO } private static void convertListVar( diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java index a6e7d593b..80d92be6b 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java @@ -22,7 +22,6 @@ public interface LunaticJsonRawDataApiPort { void save(LunaticJsonRawDataModel rawData) throws GenesisException; List getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList); - List convertRawData(List rawData, VariablesMap variablesMap); List getLunaticJsonDataByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) throws NoDataException; List getUnprocessedDataIds(); Set getUnprocessedDataQuestionnaireIds(); @@ -34,7 +33,7 @@ public interface LunaticJsonRawDataApiPort { List getRawDataByInterrogationId(String interrogationId); - DataProcessResult processRawData(String collectionInstrumentId) throws GenesisException; + DataProcessResult processRawData(String questionnaireId) throws GenesisException; Map> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since); diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index eb43894eb..49c2416ca 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -169,6 +169,7 @@ private ProcessingResultDto processRawDataForMode( List emptySurveyUnitModels = new ArrayList<>(); List surveyUnitModels = lunaticJsonRawDataConverter.convertRawDataAndCollectEmptyModels( + questionnaireId, rawData, variablesMap, emptySurveyUnitModels @@ -218,14 +219,6 @@ private VariablesMap getVariablesMap( return variablesMap; } - @Override - public List convertRawData( - List rawDataList, - VariablesMap variablesMap - ) { - return lunaticJsonRawDataConverter.convertRawData(rawDataList, variablesMap); - } - @Override public void updateProcessDates(List surveyUnitModels) { Set collectionInstrumentIds = surveyUnitModels.stream() diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index 9450eb7fc..c1e6a2034 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -5,7 +5,7 @@ import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.dto.rawdata.ProcessingResultDto; import fr.insee.genesis.controller.utils.ControllerUtils; -import fr.insee.genesis.domain.converter.rawdata.RawResponseConverter; +import fr.insee.genesis.domain.converter.rawdata.RawResponseRawDataConverter; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; @@ -22,7 +22,6 @@ import fr.insee.genesis.exceptions.NoDataException; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.modelefiliere.ModeDto; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.domain.Page; @@ -49,13 +48,13 @@ public class RawResponseService implements RawResponseApiPort { private final SurveyUnitQualityToolService surveyUnitQualityToolService; private final FileUtils fileUtils; private final Config config; - private final RawResponseConverter rawResponseConverter; + private final RawResponseRawDataConverter rawResponseRawDataConverter; @Qualifier("rawResponseMongoAdapter") private final RawResponsePersistencePort rawResponsePersistencePort; //Lombok cannot use @Qualifier - public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, SurveyUnitQualityToolService surveyUnitQualityToolService, FileUtils fileUtils, Config config, RawResponseConverter rawResponseConverter, RawResponsePersistencePort rawResponsePersistencePort) { + public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, SurveyUnitQualityToolService surveyUnitQualityToolService, FileUtils fileUtils, Config config, RawResponseRawDataConverter rawResponseRawDataConverter, RawResponsePersistencePort rawResponsePersistencePort) { this.controllerUtils = controllerUtils; this.metadataService = metadataService; this.surveyUnitService = surveyUnitService; @@ -63,7 +62,7 @@ public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadata this.surveyUnitQualityToolService = surveyUnitQualityToolService; this.fileUtils = fileUtils; this.config = config; - this.rawResponseConverter = rawResponseConverter; + this.rawResponseRawDataConverter = rawResponseRawDataConverter; this.rawResponsePersistencePort = rawResponsePersistencePort; } @@ -169,7 +168,12 @@ private ProcessingResultDto processRawResponsesForMode( List emptySurveyUnitModels = new ArrayList<>(); List surveyUnitModels = - rawResponseConverter.convertRawResponseAndCollectEmptyModels(rawResponseModels, variablesMap, emptySurveyUnitModels); + rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + collectionInstrumentId, + rawResponseModels, + variablesMap, + emptySurveyUnitModels + ); surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); surveyUnitService.saveSurveyUnits(surveyUnitModels); diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java index 2b761c481..0177aa469 100644 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java @@ -8,19 +8,28 @@ import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.parser.rawdata.LunaticJsonRawDataPayloadParser; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; @@ -29,17 +38,25 @@ @ExtendWith(MockitoExtension.class) class LunaticJsonRawDataConverterTest { - public static final LocalDateTime DATE_TIME = LocalDateTime.parse("2025-01-01T10:00:00"); + private static final LocalDateTime DATE_TIME = LocalDateTime.parse("2025-01-01T10:00:00"); + private static final String QUESTIONNAIRE_ID = "testQuestionnaire"; + + @Mock + private SurveyUnitService surveyUnitService; + @Mock private LunaticJsonRawDataPayloadParser payloadParser; + @InjectMocks + private LunaticJsonRawDataConverter converter; + @Test void shouldReturnNoSurveyUnitWhenRawDataListIsEmpty() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class); List emptySurveyUnitModels = new ArrayList<>(); List result = converter.convertRawDataAndCollectEmptyModels( + "test", List.of(), variablesMap, emptySurveyUnitModels @@ -51,7 +68,6 @@ void shouldReturnNoSurveyUnitWhenRawDataListIsEmpty() { @Test void shouldCreateCollectedAndEditedSurveyUnitsFromLegacyRawData() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class); LunaticJsonRawDataModel rawData = rawData(Map.of( @@ -60,9 +76,9 @@ void shouldCreateCollectedAndEditedSurveyUnitsFromLegacyRawData() { ) )); - try (MockedStatic rawResponseConverter = mockStatic(RawResponseConverter.class)) { + try (MockedStatic rawResponseConverter = mockStatic(RawResponseRawDataConverter.class)) { rawResponseConverter - .when(() -> RawResponseConverter.processCollectedVariable(any(), any(), any(), any(), any())) + .when(() -> RawResponseRawDataConverter.processCollectedVariable(any(), any(), any(), any(), any(), any())) .thenAnswer(invocation -> { String state = invocation.getArgument(1); List destination = invocation.getArgument(4); @@ -76,6 +92,7 @@ void shouldCreateCollectedAndEditedSurveyUnitsFromLegacyRawData() { }); List result = converter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), variablesMap ); @@ -98,7 +115,6 @@ void shouldCreateCollectedAndEditedSurveyUnitsFromLegacyRawData() { @Test void shouldConvertExternalVariablesOnlyForCollectedSurveyUnit() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); when(variablesMap.getVariable("COUNTRY").getGroupName()).thenReturn("ROOT"); @@ -115,6 +131,7 @@ void shouldConvertExternalVariablesOnlyForCollectedSurveyUnit() { List emptySurveyUnitModels = new ArrayList<>(); List result = converter.convertRawDataAndCollectEmptyModels( + QUESTIONNAIRE_ID, List.of(rawData), variablesMap, emptySurveyUnitModels @@ -144,7 +161,6 @@ void shouldConvertExternalVariablesOnlyForCollectedSurveyUnit() { @Test void shouldCollectEmptySurveyUnitsWhenNoCollectedNorExternalVariablesExist() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class); LunaticJsonRawDataModel rawData = rawData(Map.of( @@ -155,6 +171,7 @@ void shouldCollectEmptySurveyUnitsWhenNoCollectedNorExternalVariablesExist() { List emptySurveyUnitModels = new ArrayList<>(); List result = converter.convertRawDataAndCollectEmptyModels( + QUESTIONNAIRE_ID, List.of(rawData), variablesMap, emptySurveyUnitModels @@ -170,7 +187,6 @@ void shouldCollectEmptySurveyUnitsWhenNoCollectedNorExternalVariablesExist() { @Test void shouldReadFiliereRawDataFromNestedDataProperty() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); when(variablesMap.getVariable("COUNTRY").getGroupName()).thenReturn("ROOT"); @@ -185,6 +201,7 @@ void shouldReadFiliereRawDataFromNestedDataProperty() { List emptySurveyUnitModels = new ArrayList<>(); List result = converter.convertRawDataAndCollectEmptyModels( + QUESTIONNAIRE_ID, List.of(rawData), variablesMap, emptySurveyUnitModels @@ -199,7 +216,6 @@ void shouldReadFiliereRawDataFromNestedDataProperty() { @Test void shouldAssignRootScopeWhenExternalVariableIsMissingFromMetadata() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class); when(variablesMap.getVariable("UNKNOWN_VARIABLE")).thenReturn(null); @@ -210,6 +226,7 @@ void shouldAssignRootScopeWhenExternalVariableIsMissingFromMetadata() { )); List result = converter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), variablesMap ); @@ -222,15 +239,544 @@ void shouldAssignRootScopeWhenExternalVariableIsMissingFromMetadata() { .isEqualTo(Constants.ROOT_GROUP_NAME); } + @Nested + @DisplayName("Null cases tests") + class NullVariablesTests { + + @Test + @DisplayName("Should not add variable if not already existant and null in raw") + void shouldNotAddIfNull() { + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(QUESTIONNAIRE_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value("1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value("ext1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with new null variables + Map collectedVariablesMap = new LinkedHashMap<>(); + collectedVariablesMap.put("VAR1", Map.of("COLLECTED", "test")); + collectedVariablesMap.put("VAR2", null); + + Map externalVariablesMap = new LinkedHashMap<>(); + externalVariablesMap.put("EXTVAR1", "ext1"); + externalVariablesMap.put("EXTVAR2", null); + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isEqualTo("test"); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isEqualTo("ext1"); + + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should convert to non null if existant variable is null or absent") + void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPresent){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(QUESTIONNAIRE_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + if(isVariableAlreadyPresent) { + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + } + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response + Map collectedVariablesMap = new LinkedHashMap<>(); + collectedVariablesMap.put("VAR1", Map.of("COLLECTED", "1")); + + Map externalVariablesMap = new LinkedHashMap<>(); + externalVariablesMap.put("EXTVAR1", "ext1"); + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo("VAR1"); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isEqualTo("1"); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo("EXTVAR1"); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isEqualTo("ext1"); + } + + @Test + @DisplayName("Should convert to null values if non null already exists (one value)") + void shouldConvertNullValueIfNonNullOneValue(){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit with non-null variables + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(QUESTIONNAIRE_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value("1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value("ext1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with null + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + collectedVariablesMap.put("VAR1", null); + externalVariablesMap.put("EXTVAR1", null); + + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst()).isNull(); + + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst()).isNull(); + } + + @Test + @DisplayName("Should convert to null values if non null already exists (multiple iterations)") + void shouldConvertNullValueIfNonNullMultipleValues(){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit with non-null variables + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(QUESTIONNAIRE_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value("VALUE1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value("VALUE2") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value("ext1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value("ext2") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with null second iteration + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + List variablesStrings = new ArrayList<>(); + variablesStrings.add("VALUE1"); + variablesStrings.add(null); + collectedVariablesMap.put("VAR1", Map.of("COLLECTED", variablesStrings)); + + variablesStrings = new ArrayList<>(); + variablesStrings.add("ext1"); + variablesStrings.add(null); + externalVariablesMap.put("EXTVAR1", variablesStrings); + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getCollectedVariables()){ + assertThat(variableModel.iteration()).isIn(0,1); + if(variableModel.iteration().equals(1)){ + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo("VALUE1"); + } + + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getExternalVariables()) { + assertThat(variableModel.iteration()).isIn(0, 1); + if (variableModel.iteration().equals(1)) { + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo("ext1"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should keep null value if variable null or absent") + void shouldKeepNull(boolean isNewVariablesPresent){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(QUESTIONNAIRE_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + if(isNewVariablesPresent) { + collectedVariablesMap.put("VAR1", null); + externalVariablesMap.put("EXTVAR1", null); + } + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo("VAR1"); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isNull(); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo("EXTVAR1"); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isNull(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should keep null value if variable null or absent (multiple iterations)") + void shouldKeepNullIteration(boolean isNewVariablesPresent){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(QUESTIONNAIRE_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value("1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value("ext1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + + + if(isNewVariablesPresent) { + List variablesStrings = new ArrayList<>(); + variablesStrings.add("VALUE1"); + variablesStrings.add(null); + collectedVariablesMap.put("VAR1", Map.of("COLLECTED", variablesStrings)); + + variablesStrings = new ArrayList<>(); + variablesStrings.add("ext1"); + variablesStrings.add(null); + externalVariablesMap.put("EXTVAR1", variablesStrings); + } else { + collectedVariablesMap.put("VAR1", Map.of("COLLECTED", List.of("VALUE1"))); + externalVariablesMap.put("EXTVAR1", List.of("ext1")); + } + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo("VAR1"); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isNull(); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo("EXTVAR1"); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isNull(); + } + } + private LunaticJsonRawDataModel rawData(Map data) { - LunaticJsonRawDataModel rawData = mock(LunaticJsonRawDataModel.class); - - when(rawData.questionnaireId()).thenReturn("questionnaire-id"); - when(rawData.mode()).thenReturn(Mode.valueOf("WEB")); - when(rawData.interrogationId()).thenReturn("interrogation-id"); - when(rawData.idUE()).thenReturn("survey-unit-id"); - when(rawData.recordDate()).thenReturn(DATE_TIME); - when(rawData.data()).thenReturn(data); + LunaticJsonRawDataModel rawData = LunaticJsonRawDataModel.builder() + .questionnaireId("questionnaire-id") + .mode(Mode.WEB) + .interrogationId("interrogation-id") + .idUE("survey-unit-id") + .recordDate(DATE_TIME) + .data(data) + .build(); when(payloadParser.getValidationDate(rawData)).thenReturn(DATE_TIME); when(payloadParser.getIsCapturedIndirectly(rawData)).thenReturn(false); diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java new file mode 100644 index 000000000..6dba133dc --- /dev/null +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java @@ -0,0 +1,89 @@ +package fr.insee.genesis.domain.converter.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; + +@ExtendWith(MockitoExtension.class) +class RawDataConverterTest { + + @Mock + private SurveyUnitService surveyUnitService; + + @InjectMocks + private final RawDataConverter rawDataConverterTestImpl = new RawDataConverter() { + @Override + protected Map getLastSurveyUnitModels(String questionnaireOrCollectionInstrumentId, List interrogationIds) { + return super.getLastSurveyUnitModels(questionnaireOrCollectionInstrumentId, interrogationIds); + } + }; + + + @Nested + @DisplayName("getLastSurveyUnitModels tests") + class getLastSurveyUnitModelsTests{ + @Test + @DisplayName("Should return map with interrogation ids as key") + void getLastSurveyUnitModelsTest() { + //GIVEN + String questionnaireId = "questionnaire"; + List surveyUnitModels = + List.of( + SurveyUnitModel.builder() + .collectionInstrumentId(questionnaireId) + .interrogationId("Interrogation1") + .build(), + SurveyUnitModel.builder() + .collectionInstrumentId(questionnaireId) + .interrogationId("Interrogation2") + .build() + ); + + + Mockito.when(surveyUnitService.findLatestByInterrogationIds( + eq(questionnaireId), anySet()) + ).thenReturn(surveyUnitModels); + + //WHEN + Map resultMap = rawDataConverterTestImpl.getLastSurveyUnitModels( + questionnaireId, + List.of("Interrogation1","Interrogation2") + ); + + + //THEN + //Service call with good arguments + @SuppressWarnings("unchecked") + ArgumentCaptor> setArgumentCaptor = ArgumentCaptor.forClass(Set.class); + Mockito.verify(surveyUnitService, Mockito.times(1)).findLatestByInterrogationIds( + eq(questionnaireId), + setArgumentCaptor.capture() + ); + Assertions.assertThat(setArgumentCaptor.getValue()).containsExactlyInAnyOrder( + "Interrogation1","Interrogation2" + ); + + //Check resulted map key and values content + Assertions.assertThat(resultMap).containsOnlyKeys("Interrogation1","Interrogation2"); + Assertions.assertThat(resultMap.get("Interrogation1").getInterrogationId()).isEqualTo("Interrogation1"); + Assertions.assertThat(resultMap.get("Interrogation2").getInterrogationId()).isEqualTo("Interrogation2"); + } + } +} \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverterTest.java deleted file mode 100644 index 7f3460d84..000000000 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverterTest.java +++ /dev/null @@ -1,314 +0,0 @@ -package fr.insee.genesis.domain.converter.rawdata; - -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; -import fr.insee.genesis.domain.model.surveyunit.DataState; -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; -import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser; -import fr.insee.modelefiliere.RawResponseDto; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.tuple; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class RawResponseConverterTest { - - @Mock - private RawResponsePayloadParser rawResponsePayloadParser; - - @Test - void shouldReturnNoSurveyUnitWhenRawResponseListIsEmpty() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class); - List emptySurveyUnitModels = new ArrayList<>(); - - List result = converter.convertRawResponseAndCollectEmptyModels( - List.of(), - variablesMap, - emptySurveyUnitModels - ); - - assertThat(result).isEmpty(); - assertThat(emptySurveyUnitModels).isEmpty(); - } - - @Test - void shouldCreateCollectedAndEditedSurveyUnitsFromCollectedPayloadStates() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); - - when(variablesMap.getVariable("FIRST_NAME").getGroupName()).thenReturn(Constants.ROOT_GROUP_NAME); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - collectedVariables( - Map.of( - "COLLECTED", "Alice", - "EDITED", "Alicia" - ) - ), - Map.of() - )); - - List result = converter.convertRawResponse( - List.of(rawResponseModel), - variablesMap - ); - - assertThat(result).hasSize(2); - - assertThat(result) - .extracting(SurveyUnitModel::getState) - .containsExactly(DataState.COLLECTED, DataState.EDITED); - - assertThat(result.get(0).getCollectedVariables()) - .extracting("varId", "value", "iteration", "scope") - .containsExactly(tuple("FIRST_NAME", "Alice", 1, Constants.ROOT_GROUP_NAME)); - - assertThat(result.get(1).getCollectedVariables()) - .extracting("varId", "value", "iteration", "scope") - .containsExactly(tuple("FIRST_NAME", "Alicia", 1, Constants.ROOT_GROUP_NAME)); - } - - @Test - void shouldConvertExternalVariablesOnlyForCollectedSurveyUnit() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); - - when(variablesMap.getVariable("COUNTRY").getGroupName()).thenReturn(Constants.ROOT_GROUP_NAME); - when(variablesMap.getVariable("CHILDREN").getGroupName()).thenReturn("HOUSEHOLD"); - - Map externalVariables = new LinkedHashMap<>(); - externalVariables.put("COUNTRY", "FR"); - externalVariables.put("CHILDREN", List.of("Anna", "", "Paul")); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(), - externalVariables - )); - - List emptySurveyUnitModels = new ArrayList<>(); - - List result = converter.convertRawResponseAndCollectEmptyModels( - List.of(rawResponseModel), - variablesMap, - emptySurveyUnitModels - ); - - assertThat(result).hasSize(1); - - SurveyUnitModel collectedSurveyUnit = result.getFirst(); - - assertThat(collectedSurveyUnit.getState()).isEqualTo(DataState.COLLECTED); - assertThat(collectedSurveyUnit.getCollectedVariables()).isEmpty(); - - assertThat(collectedSurveyUnit.getExternalVariables()) - .extracting("varId", "value", "iteration", "scope") - .containsExactly( - tuple("COUNTRY", "FR", 1, Constants.ROOT_GROUP_NAME), - tuple("CHILDREN", "Anna", 1, "HOUSEHOLD"), - tuple("CHILDREN", "Paul", 3, "HOUSEHOLD") - ); - - assertThat(emptySurveyUnitModels) - .hasSize(1) - .extracting(SurveyUnitModel::getState) - .containsExactly(DataState.EDITED); - } - - @Test - void shouldCollectEmptySurveyUnitsWhenNoVariableExists() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(), - Map.of() - )); - - List emptySurveyUnitModels = new ArrayList<>(); - - List result = converter.convertRawResponseAndCollectEmptyModels( - List.of(rawResponseModel), - variablesMap, - emptySurveyUnitModels - ); - - assertThat(result).isEmpty(); - - assertThat(emptySurveyUnitModels) - .hasSize(2) - .extracting(SurveyUnitModel::getState) - .containsExactly(DataState.COLLECTED, DataState.EDITED); - } - - @Test - void shouldAssignRootScopeWhenVariableIsMissingFromMetadata() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class); - - when(variablesMap.getVariable("UNKNOWN_VARIABLE")).thenReturn(null); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(), - Map.of("UNKNOWN_VARIABLE", "value") - )); - - List result = converter.convertRawResponse( - List.of(rawResponseModel), - variablesMap - ); - - assertThat(result).hasSize(1); - - assertThat(result.getFirst().getExternalVariables()) - .singleElement() - .extracting("scope") - .isEqualTo(Constants.ROOT_GROUP_NAME); - } - - @Test - void shouldSetQuestionnaireStateWhenPayloadContainsValidState() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(), - Map.of() - )); - - when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) - .thenReturn(RawResponseDto.QuestionnaireStateEnum.FINISHED.name()); - - List emptySurveyUnitModels = new ArrayList<>(); - - converter.convertRawResponseAndCollectEmptyModels( - List.of(rawResponseModel), - variablesMap, - emptySurveyUnitModels - ); - - assertThat(emptySurveyUnitModels) - .extracting(SurveyUnitModel::getQuestionnaireState) - .containsOnly(RawResponseDto.QuestionnaireStateEnum.FINISHED); - } - - @Test - void shouldKeepQuestionnaireStateNullWhenPayloadContainsInvalidState() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(), - Map.of() - )); - - when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) - .thenReturn("INVALID_STATE"); - - List emptySurveyUnitModels = new ArrayList<>(); - - converter.convertRawResponseAndCollectEmptyModels( - List.of(rawResponseModel), - variablesMap, - emptySurveyUnitModels - ); - - assertThat(emptySurveyUnitModels) - .extracting(SurveyUnitModel::getQuestionnaireState) - .containsOnlyNulls(); - } - - @Test - void shouldConvertPairwiseCollectedVariables() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); - - when(variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1)).thenReturn(true); - when(variablesMap.getVariable(Constants.PAIRWISE_PREFIX + 1).getGroupName()).thenReturn("PAIRWISE_GROUP"); - - List> pairwiseValues = List.of( - List.of("1", "", "3"), - List.of() - ); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(Constants.PAIRWISES, Map.of("COLLECTED", pairwiseValues)), - Map.of() - )); - - List result = converter.convertRawResponse( - List.of(rawResponseModel), - variablesMap - ); - - SurveyUnitModel collectedSurveyUnit = result.getFirst(); - - assertThat(collectedSurveyUnit.getCollectedVariables()) - .hasSize(2 * (Constants.MAX_LINKS_ALLOWED - 1)); - - assertThat(collectedSurveyUnit.getCollectedVariables()) - .extracting("varId", "value", "iteration", "scope", "parentId") - .contains( - tuple(Constants.PAIRWISE_PREFIX + 1, "1", 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), - tuple(Constants.PAIRWISE_PREFIX + 2, Constants.SAME_AXIS_VALUE, 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), - tuple(Constants.PAIRWISE_PREFIX + 3, "3", 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), - tuple(Constants.PAIRWISE_PREFIX + 1, Constants.NO_PAIRWISE_VALUE, 2, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME) - ); - } - - private RawResponseModel rawResponseModel(Map payload) { - RawResponseModel rawResponseModel = mock(RawResponseModel.class); - - when(rawResponseModel.collectionInstrumentId()).thenReturn("collection-instrument-id"); - when(rawResponseModel.mode()).thenReturn(Mode.valueOf("WEB")); - when(rawResponseModel.interrogationId()).thenReturn("interrogation-id"); - when(rawResponseModel.recordDate()).thenReturn(LocalDateTime.parse("2025-01-01T10:00:00")); - when(rawResponseModel.payload()).thenReturn(payload); - - when(rawResponsePayloadParser.getStringField(rawResponseModel, "majorModelVersion")) - .thenReturn("1"); - when(rawResponsePayloadParser.getStringField(rawResponseModel, "usualSurveyUnitId")) - .thenReturn("survey-unit-id"); - when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) - .thenReturn(null); - when(rawResponsePayloadParser.getValidationDate(rawResponseModel)) - .thenReturn(LocalDateTime.parse("2025-01-02T10:00:00")); - when(rawResponsePayloadParser.getIsCapturedIndirectly(rawResponseModel)) - .thenReturn(false); - - return rawResponseModel; - } - - private static Map payloadWith( - Map collectedVariables, - Map externalVariables - ) { - return Map.of( - "data", Map.of( - "COLLECTED", collectedVariables, - "EXTERNAL", externalVariables - ) - ); - } - - private static Map collectedVariables( - Map states - ) { - return Map.of("FIRST_NAME", states); - } -} \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java new file mode 100644 index 000000000..20406b4b4 --- /dev/null +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java @@ -0,0 +1,868 @@ +package fr.insee.genesis.domain.converter.rawdata; + +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; +import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.modelefiliere.RawResponseDto; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class RawResponseRawDataConverterTest { + + private static final String COLLECTION_INSTRUMENT_ID = "collection-instrument-id"; + private static final Mode MODE = Mode.WEB; + + + @Mock + private RawResponsePayloadParser rawResponsePayloadParser; + + @Mock + private SurveyUnitService surveyUnitService; + + @InjectMocks + private RawResponseRawDataConverter rawResponseRawDataConverter; + + @Test + void shouldReturnNoSurveyUnitWhenRawResponseListIsEmpty() { + VariablesMap variablesMap = mock(VariablesMap.class); + List emptySurveyUnitModels = new ArrayList<>(); + + List result = rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + COLLECTION_INSTRUMENT_ID, + List.of(), + variablesMap, + emptySurveyUnitModels + ); + + assertThat(result).isEmpty(); + assertThat(emptySurveyUnitModels).isEmpty(); + } + + @Test + void shouldCreateCollectedAndEditedSurveyUnitsFromCollectedPayloadStates() { + VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); + + when(variablesMap.getVariable("FIRST_NAME").getGroupName()).thenReturn(Constants.ROOT_GROUP_NAME); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + collectedVariables( + Map.of( + "COLLECTED", "Alice", + "EDITED", "Alicia" + ) + ), + Map.of() + )); + + List result = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap + ); + + assertThat(result).hasSize(2); + + assertThat(result) + .extracting(SurveyUnitModel::getState) + .containsExactly(DataState.COLLECTED, DataState.EDITED); + + assertThat(result.get(0).getCollectedVariables()) + .extracting("varId", "value", "iteration", "scope") + .containsExactly(tuple("FIRST_NAME", "Alice", 1, Constants.ROOT_GROUP_NAME)); + + assertThat(result.get(1).getCollectedVariables()) + .extracting("varId", "value", "iteration", "scope") + .containsExactly(tuple("FIRST_NAME", "Alicia", 1, Constants.ROOT_GROUP_NAME)); + } + + @Test + void shouldConvertExternalVariablesOnlyForCollectedSurveyUnit() { + VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); + + when(variablesMap.getVariable("COUNTRY").getGroupName()).thenReturn(Constants.ROOT_GROUP_NAME); + when(variablesMap.getVariable("CHILDREN").getGroupName()).thenReturn("HOUSEHOLD"); + + Map externalVariables = new LinkedHashMap<>(); + externalVariables.put("COUNTRY", "FR"); + externalVariables.put("CHILDREN", List.of("Anna", "", "Paul")); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(), + externalVariables + )); + + List emptySurveyUnitModels = new ArrayList<>(); + + List result = rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap, + emptySurveyUnitModels + ); + + assertThat(result).hasSize(1); + + SurveyUnitModel collectedSurveyUnit = result.getFirst(); + + assertThat(collectedSurveyUnit.getState()).isEqualTo(DataState.COLLECTED); + assertThat(collectedSurveyUnit.getCollectedVariables()).isEmpty(); + + assertThat(collectedSurveyUnit.getExternalVariables()) + .extracting("varId", "value", "iteration", "scope") + .containsExactly( + tuple("COUNTRY", "FR", 1, Constants.ROOT_GROUP_NAME), + tuple("CHILDREN", "Anna", 1, "HOUSEHOLD"), + tuple("CHILDREN", "Paul", 3, "HOUSEHOLD") + ); + + assertThat(emptySurveyUnitModels) + .hasSize(1) + .extracting(SurveyUnitModel::getState) + .containsExactly(DataState.EDITED); + } + + @Test + void shouldCollectEmptySurveyUnitsWhenNoVariableExists() { + VariablesMap variablesMap = mock(VariablesMap.class); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(), + Map.of() + )); + + List emptySurveyUnitModels = new ArrayList<>(); + + List result = rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap, + emptySurveyUnitModels + ); + + assertThat(result).isEmpty(); + + assertThat(emptySurveyUnitModels) + .hasSize(2) + .extracting(SurveyUnitModel::getState) + .containsExactly(DataState.COLLECTED, DataState.EDITED); + } + + @Test + void shouldAssignRootScopeWhenVariableIsMissingFromMetadata() { + VariablesMap variablesMap = mock(VariablesMap.class); + + when(variablesMap.getVariable("UNKNOWN_VARIABLE")).thenReturn(null); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(), + Map.of("UNKNOWN_VARIABLE", "value") + )); + + List result = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap + ); + + assertThat(result).hasSize(1); + + assertThat(result.getFirst().getExternalVariables()) + .singleElement() + .extracting("scope") + .isEqualTo(Constants.ROOT_GROUP_NAME); + } + + @Test + void shouldSetQuestionnaireStateWhenPayloadContainsValidState() { + VariablesMap variablesMap = mock(VariablesMap.class); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(), + Map.of() + )); + + when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) + .thenReturn(RawResponseDto.QuestionnaireStateEnum.FINISHED.name()); + + List emptySurveyUnitModels = new ArrayList<>(); + + rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap, + emptySurveyUnitModels + ); + + assertThat(emptySurveyUnitModels) + .extracting(SurveyUnitModel::getQuestionnaireState) + .containsOnly(RawResponseDto.QuestionnaireStateEnum.FINISHED); + } + + @Test + void shouldKeepQuestionnaireStateNullWhenPayloadContainsInvalidState() { + VariablesMap variablesMap = mock(VariablesMap.class); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(), + Map.of() + )); + + when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) + .thenReturn("INVALID_STATE"); + + List emptySurveyUnitModels = new ArrayList<>(); + + rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap, + emptySurveyUnitModels + ); + + assertThat(emptySurveyUnitModels) + .extracting(SurveyUnitModel::getQuestionnaireState) + .containsOnlyNulls(); + } + + @Test + void shouldConvertPairwiseCollectedVariables() { + RawResponseRawDataConverter converter = new RawResponseRawDataConverter( + surveyUnitService, rawResponsePayloadParser + ); + VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); + + when(variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1)).thenReturn(true); + when(variablesMap.getVariable(Constants.PAIRWISE_PREFIX + 1).getGroupName()).thenReturn("PAIRWISE_GROUP"); + + List> pairwiseValues = List.of( + List.of("1", "", "3"), + List.of() + ); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(Constants.PAIRWISES, Map.of("COLLECTED", pairwiseValues)), + Map.of() + )); + + List result = converter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap + ); + + SurveyUnitModel collectedSurveyUnit = result.getFirst(); + + assertThat(collectedSurveyUnit.getCollectedVariables()) + .hasSize(2 * (Constants.MAX_LINKS_ALLOWED - 1)); + + assertThat(collectedSurveyUnit.getCollectedVariables()) + .extracting("varId", "value", "iteration", "scope", "parentId") + .contains( + tuple(Constants.PAIRWISE_PREFIX + 1, "1", 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), + tuple(Constants.PAIRWISE_PREFIX + 2, Constants.SAME_AXIS_VALUE, 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), + tuple(Constants.PAIRWISE_PREFIX + 3, "3", 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), + tuple(Constants.PAIRWISE_PREFIX + 1, Constants.NO_PAIRWISE_VALUE, 2, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME) + ); + } + + @Nested + @DisplayName("Null cases tests") + class NullVariablesTests { + + @Test + @DisplayName("Should not add variable if not already existant and null in raw") + void shouldNotAddIfNull(){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value("1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value("ext1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with new null variables + Map collectedVariablesMap = new LinkedHashMap<>(); + collectedVariablesMap.put("VAR1", Map.of("COLLECTED", "test")); + collectedVariablesMap.put("VAR2", null); + + Map externalVariablesMap = new LinkedHashMap<>(); + externalVariablesMap.put("EXTVAR1", "ext1"); + externalVariablesMap.put("EXTVAR2", null); + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + collectedVariablesMap, + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isEqualTo("test"); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isEqualTo("ext1"); + + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should convert to non null if existant variable is null or absent") + void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPresent){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + if(isVariableAlreadyPresent) { + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + } + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response + Map collectedVariablesMap = new LinkedHashMap<>(); + collectedVariablesMap.put("VAR1", Map.of("COLLECTED", "1")); + + Map externalVariablesMap = new LinkedHashMap<>(); + externalVariablesMap.put("EXTVAR1", "ext1"); + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + collectedVariablesMap, + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo("VAR1"); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isEqualTo("1"); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo("EXTVAR1"); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isEqualTo("ext1"); + } + + @Test + @DisplayName("Should convert to null values if non null already exists (one value)") + void shouldConvertNullValueIfNonNullOneValue(){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit with non-null variables + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value("1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value("ext1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with null + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + collectedVariablesMap.put("VAR1", null); + externalVariablesMap.put("EXTVAR1", null); + + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + Map.of("COLLECTED", collectedVariablesMap), + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst()).isNull(); + + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst()).isNull(); + } + + @Test + @DisplayName("Should convert to null values if non null already exists (multiple iterations)") + void shouldConvertNullValueIfNonNullMultipleValues(){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit with non-null variables + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value("VALUE1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value("VALUE2") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value("ext1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value("ext2") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with null second iteration + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + List variablesStrings = new ArrayList<>(); + variablesStrings.add("VALUE1"); + variablesStrings.add(null); + collectedVariablesMap.put("VAR1", Map.of("COLLECTED", variablesStrings)); + + variablesStrings = new ArrayList<>(); + variablesStrings.add("ext1"); + variablesStrings.add(null); + externalVariablesMap.put("EXTVAR1", variablesStrings); + + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + Map.of("COLLECTED", collectedVariablesMap), + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getCollectedVariables()){ + assertThat(variableModel.iteration()).isIn(0,1); + if(variableModel.iteration().equals(1)){ + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo("VALUE1"); + } + + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getExternalVariables()) { + assertThat(variableModel.iteration()).isIn(0, 1); + if (variableModel.iteration().equals(1)) { + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo("ext1"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should keep null value if variable null or absent") + void shouldKeepNull(boolean isNewVariablesPresent){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + if(isNewVariablesPresent) { + collectedVariablesMap.put("VAR1", null); + externalVariablesMap.put("EXTVAR1", null); + } + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + Map.of("COLLECTED", collectedVariablesMap), + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo("VAR1"); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isNull(); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo("EXTVAR1"); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isNull(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should keep null value if variable null or absent (multiple iterations)") + void shouldKeepNullIteration(boolean isNewVariablesPresent){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + String interrogationId = "testInterrogation"; + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .interrogationId(interrogationId) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value("1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId("VAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value("ext1") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId("EXTVAR1") + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + + + if(isNewVariablesPresent) { + List variablesStrings = new ArrayList<>(); + variablesStrings.add("VALUE1"); + variablesStrings.add(null); + collectedVariablesMap.put("VAR1", Map.of("COLLECTED", variablesStrings)); + + variablesStrings = new ArrayList<>(); + variablesStrings.add("ext1"); + variablesStrings.add(null); + externalVariablesMap.put("EXTVAR1", variablesStrings); + } else { + collectedVariablesMap.put("VAR1", Map.of("COLLECTED", List.of("VALUE1"))); + externalVariablesMap.put("EXTVAR1", List.of("ext1")); + } + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + Map.of("COLLECTED", collectedVariablesMap), + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo("VAR1"); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isNull(); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo("EXTVAR1"); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isNull(); + } + } + + + //UTILS + private RawResponseModel rawResponseModel(Map payload) { + RawResponseModel rawResponseModel = RawResponseModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .mode(MODE) + .interrogationId("interrogation-id") + .recordDate(LocalDateTime.parse("2025-01-01T10:00:00")) + .payload(payload) + .build(); + + when(rawResponsePayloadParser.getStringField(rawResponseModel, "majorModelVersion")) + .thenReturn("1"); + when(rawResponsePayloadParser.getStringField(rawResponseModel, "usualSurveyUnitId")) + .thenReturn("survey-unit-id"); + when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) + .thenReturn(null); + when(rawResponsePayloadParser.getValidationDate(rawResponseModel)) + .thenReturn(LocalDateTime.parse("2025-01-02T10:00:00")); + when(rawResponsePayloadParser.getIsCapturedIndirectly(rawResponseModel)) + .thenReturn(false); + + return rawResponseModel; + } + + private static Map payloadWith( + Map collectedVariables, + Map externalVariables + ) { + return Map.of( + "data", Map.of( + "COLLECTED", collectedVariables, + "EXTERNAL", externalVariables + ) + ); + } + + private static Map collectedVariables( + Map states + ) { + return Map.of("FIRST_NAME", states); + } +} \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index ce9d62cbf..91164a551 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -98,7 +98,7 @@ class LunaticJsonRawDataServiceTest { @BeforeEach void init() { - lunaticJsonRawDataConverter = new LunaticJsonRawDataConverter(new LunaticJsonRawDataPayloadParser()); + lunaticJsonRawDataConverter = new LunaticJsonRawDataConverter(surveyUnitService, new LunaticJsonRawDataPayloadParser()); service = new LunaticJsonRawDataService( controllerUtils, @@ -291,7 +291,9 @@ class ConvertRawDataTests { @DisplayName("Empty raw data list returns empty list") void emptyRawDataList_returnsEmpty() { //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(), new VariablesMap() + ); //THEN assertThat(result).isEmpty(); @@ -307,6 +309,7 @@ void noVariables_rawDataIgnored() { // WHEN List result = lunaticJsonRawDataConverter.convertRawDataAndCollectEmptyModels( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap(), emptySurveyUnitModels @@ -330,7 +333,9 @@ void withCollectedVariable_producesBothDataStates() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(collected); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); //THEN // Expect 1 COLLECTED (VAR1 has a value) and 0 EDITED (EDITED value is null → empty) @@ -355,7 +360,9 @@ void withEditedVariable_producesEditedModel() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(collected); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); //THEN long editedCount = result.stream() @@ -381,7 +388,9 @@ void filiereModelType_detected() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(outerData); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); // Should not throw — FILIERE path is followed TODO More asserts assertThat(result).isNotNull(); @@ -412,7 +421,9 @@ void optionalFields_mappedCorrectly() { .build(); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); //THEN assertThat(result).isNotEmpty(); @@ -447,7 +458,9 @@ void malformedValidationDate_fallsBackToNull() { .build(); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); //THEN assertThat(result).isNotEmpty(); result.stream() @@ -468,7 +481,9 @@ void arrayValues_convertedToMultipleIterations() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(collected); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); //THEN SurveyUnitModel collectedModel = result.stream() diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java index 2ce81f1f1..cc81ecf0d 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java @@ -4,7 +4,7 @@ import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.TestConstants; import fr.insee.genesis.controller.utils.ControllerUtils; -import fr.insee.genesis.domain.converter.rawdata.RawResponseConverter; +import fr.insee.genesis.domain.converter.rawdata.RawResponseRawDataConverter; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; @@ -69,7 +69,7 @@ class RawResponseServiceUnitTest { @Mock private SurveyUnitQualityToolService surveyUnitQualityToolService; - private RawResponseConverter rawResponseConverter; + private RawResponseRawDataConverter rawResponseRawDataConverter; @Mock static DataProcessingContextService dataProcessingContextService; @@ -80,7 +80,7 @@ class RawResponseServiceUnitTest { @BeforeEach void init() { - rawResponseConverter = new RawResponseConverter(new RawResponsePayloadParser()); + rawResponseRawDataConverter = new RawResponseRawDataConverter(surveyUnitService, new RawResponsePayloadParser()); rawResponseService = new RawResponseService( controllerUtils, @@ -90,7 +90,7 @@ void init() { surveyUnitQualityToolService, new FileUtils(TestConstants.getConfigStub()), TestConstants.getConfigStub(), - rawResponseConverter, + rawResponseRawDataConverter, rawResponsePersistencePort ); } @@ -133,7 +133,7 @@ void getUnprocessedCollectionInstrumentIds_shouldnt_return_if_no_spec() { surveyUnitQualityToolService, new FileUtils(TestConstants.getConfigStub()), TestConstants.getConfigStub(), - rawResponseConverter, + rawResponseRawDataConverter, rawResponsePersistencePort ); @@ -493,7 +493,9 @@ void convertRawResponse_shouldConvertCollectedVariable() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID,rawResponses, variablesMap + ); // THEN Assertions.assertThat(result).hasSize(1); // only COLLECTED state (EDITED has no data) @@ -510,7 +512,9 @@ void convertRawResponse_shouldConvertEditedVariable() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, rawResponses, variablesMap + ); // THEN // On attend 1 modèle EDITED avec des variables @@ -530,7 +534,9 @@ void convertRawResponse_shouldConvertExternalVariables() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, rawResponses, variablesMap + ); // THEN List collectedModels = result.stream() @@ -549,7 +555,8 @@ void convertRawResponse_shouldIgnoreEmptyResponse() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, rawResponses, variablesMap); // THEN Assertions.assertThat(result).isEmpty(); @@ -563,7 +570,9 @@ void convertRawResponse_shouldHandleListValues() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, rawResponses, variablesMap + ); // THEN List collectedModels = result.stream() @@ -585,7 +594,9 @@ void convertRawResponse_shouldSkipNullOrEmptyListValues() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, rawResponses, variablesMap + ); // THEN List collectedModels = result.stream() From 3dff31f236874326028483a619920e317fc03e02 Mon Sep 17 00:00:00 2001 From: Alexis Szmundy Date: Thu, 4 Jun 2026 11:07:27 +0200 Subject: [PATCH 03/12] refactor: converter tests --- .../LunaticJsonRawDataConverterTest.java | 326 +++++++---------- .../RawResponseRawDataConverterTest.java | 342 +++++++----------- 2 files changed, 257 insertions(+), 411 deletions(-) diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java index 0177aa469..feee30065 100644 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java @@ -243,6 +243,13 @@ void shouldAssignRootScopeWhenExternalVariableIsMissingFromMetadata() { @DisplayName("Null cases tests") class NullVariablesTests { + //NullVariablesTests constants + private static final String INTERROGATION_ID = "testInterrogation"; + private static final String COLLECTED_VARIABLE_NAME = "VAR1"; + private static final String COLLECTED_VARIABLE_VALUE = "test"; + private static final String EXTERNAL_VARIABLE_NAME = "EXTVAR1"; + private static final String EXTERNAL_VARIABLE_VALUE = "ext1"; + @Test @DisplayName("Should not add variable if not already existant and null in raw") void shouldNotAddIfNull() { @@ -250,32 +257,7 @@ void shouldNotAddIfNull() { VariablesMap variablesMap = mock(VariablesMap.class); //Already existing survey unit - String interrogationId = "testInterrogation"; - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .collectionInstrumentId(QUESTIONNAIRE_ID) - .interrogationId(interrogationId) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - - surveyUnitModel.getCollectedVariables().add( - VariableModel.builder() - .varId("VAR1") - .value("1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getExternalVariables().add( - VariableModel.builder() - .varId("EXTVAR1") - .value("ext1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) .thenReturn(List.of( @@ -284,11 +266,11 @@ void shouldNotAddIfNull() { //Raw response with new null variables Map collectedVariablesMap = new LinkedHashMap<>(); - collectedVariablesMap.put("VAR1", Map.of("COLLECTED", "test")); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", COLLECTED_VARIABLE_VALUE)); collectedVariablesMap.put("VAR2", null); Map externalVariablesMap = new LinkedHashMap<>(); - externalVariablesMap.put("EXTVAR1", "ext1"); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, EXTERNAL_VARIABLE_VALUE); externalVariablesMap.put("EXTVAR2", null); LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( @@ -308,14 +290,7 @@ void shouldNotAddIfNull() { ); //THEN - assertThat(surveyUnitModels).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .value()).isEqualTo("test"); - assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .value()).isEqualTo("ext1"); - + assertNonNullVariables(surveyUnitModels); } @ParameterizedTest @@ -326,10 +301,9 @@ void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPrese VariablesMap variablesMap = mock(VariablesMap.class); //Already existing survey unit - String interrogationId = "testInterrogation"; SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .collectionInstrumentId(QUESTIONNAIRE_ID) - .interrogationId(interrogationId) + .interrogationId(INTERROGATION_ID) .collectedVariables(new ArrayList<>()) .externalVariables(new ArrayList<>()) .build(); @@ -337,7 +311,7 @@ void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPrese if(isVariableAlreadyPresent) { surveyUnitModel.getCollectedVariables().add( VariableModel.builder() - .varId("VAR1") + .varId(COLLECTED_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -346,7 +320,7 @@ void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPrese ); surveyUnitModel.getExternalVariables().add( VariableModel.builder() - .varId("EXTVAR1") + .varId(EXTERNAL_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -362,10 +336,10 @@ void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPrese //Raw response Map collectedVariablesMap = new LinkedHashMap<>(); - collectedVariablesMap.put("VAR1", Map.of("COLLECTED", "1")); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", COLLECTED_VARIABLE_VALUE)); Map externalVariablesMap = new LinkedHashMap<>(); - externalVariablesMap.put("EXTVAR1", "ext1"); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, EXTERNAL_VARIABLE_VALUE); LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( Map.of( @@ -384,17 +358,7 @@ void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPrese ); //THEN - assertThat(surveyUnitModels).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .varId()).isEqualTo("VAR1"); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .value()).isEqualTo("1"); - assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .varId()).isEqualTo("EXTVAR1"); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .value()).isEqualTo("ext1"); + assertNonNullVariables(surveyUnitModels); } @Test @@ -404,32 +368,7 @@ void shouldConvertNullValueIfNonNullOneValue(){ VariablesMap variablesMap = mock(VariablesMap.class); //Already existing survey unit with non-null variables - String interrogationId = "testInterrogation"; - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .collectionInstrumentId(QUESTIONNAIRE_ID) - .interrogationId(interrogationId) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - - surveyUnitModel.getCollectedVariables().add( - VariableModel.builder() - .varId("VAR1") - .value("1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getExternalVariables().add( - VariableModel.builder() - .varId("EXTVAR1") - .value("ext1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) .thenReturn(List.of( @@ -440,8 +379,8 @@ void shouldConvertNullValueIfNonNullOneValue(){ Map collectedVariablesMap = new LinkedHashMap<>(); Map externalVariablesMap = new LinkedHashMap<>(); - collectedVariablesMap.put("VAR1", null); - externalVariablesMap.put("EXTVAR1", null); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, null); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, null); LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( @@ -461,12 +400,7 @@ void shouldConvertNullValueIfNonNullOneValue(){ ); //THEN - assertThat(surveyUnitModels).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst()).isNull(); - - assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst()).isNull(); + assertNullVariables(surveyUnitModels); } @Test @@ -475,27 +409,11 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ //GIVEN VariablesMap variablesMap = mock(VariablesMap.class); - //Already existing survey unit with non-null variables - String interrogationId = "testInterrogation"; - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .collectionInstrumentId(QUESTIONNAIRE_ID) - .interrogationId(interrogationId) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - + //Already existing survey unit with non-null variables and new iterations + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); surveyUnitModel.getCollectedVariables().add( VariableModel.builder() - .varId("VAR1") - .value("VALUE1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getCollectedVariables().add( - VariableModel.builder() - .varId("VAR1") + .varId(COLLECTED_VARIABLE_NAME) .value("VALUE2") .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -504,16 +422,7 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ ); surveyUnitModel.getExternalVariables().add( VariableModel.builder() - .varId("EXTVAR1") - .value("ext1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getExternalVariables().add( - VariableModel.builder() - .varId("EXTVAR1") + .varId(EXTERNAL_VARIABLE_NAME) .value("ext2") .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -531,14 +440,14 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ Map externalVariablesMap = new LinkedHashMap<>(); List variablesStrings = new ArrayList<>(); - variablesStrings.add("VALUE1"); + variablesStrings.add(COLLECTED_VARIABLE_VALUE); variablesStrings.add(null); - collectedVariablesMap.put("VAR1", Map.of("COLLECTED", variablesStrings)); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", variablesStrings)); variablesStrings = new ArrayList<>(); - variablesStrings.add("ext1"); + variablesStrings.add(EXTERNAL_VARIABLE_VALUE); variablesStrings.add(null); - externalVariablesMap.put("EXTVAR1", variablesStrings); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, variablesStrings); LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( Map.of( @@ -557,26 +466,7 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ ); //THEN - assertThat(surveyUnitModels).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(2); - for(VariableModel variableModel : surveyUnitModels.getFirst().getCollectedVariables()){ - assertThat(variableModel.iteration()).isIn(0,1); - if(variableModel.iteration().equals(1)){ - assertThat(variableModel.value()).isNull(); - continue; - } - assertThat(variableModel.value()).isEqualTo("VALUE1"); - } - - assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(2); - for(VariableModel variableModel : surveyUnitModels.getFirst().getExternalVariables()) { - assertThat(variableModel.iteration()).isIn(0, 1); - if (variableModel.iteration().equals(1)) { - assertThat(variableModel.value()).isNull(); - continue; - } - assertThat(variableModel.value()).isEqualTo("ext1"); - } + assertSecondIterationNull(surveyUnitModels); } @ParameterizedTest @@ -586,18 +476,16 @@ void shouldKeepNull(boolean isNewVariablesPresent){ //GIVEN VariablesMap variablesMap = mock(VariablesMap.class); - //Already existing survey unit - String interrogationId = "testInterrogation"; + //Already existing survey unit with null SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .collectionInstrumentId(QUESTIONNAIRE_ID) - .interrogationId(interrogationId) + .interrogationId(INTERROGATION_ID) .collectedVariables(new ArrayList<>()) .externalVariables(new ArrayList<>()) .build(); - surveyUnitModel.getCollectedVariables().add( VariableModel.builder() - .varId("VAR1") + .varId(COLLECTED_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -606,7 +494,7 @@ void shouldKeepNull(boolean isNewVariablesPresent){ ); surveyUnitModel.getExternalVariables().add( VariableModel.builder() - .varId("EXTVAR1") + .varId(EXTERNAL_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -624,8 +512,8 @@ void shouldKeepNull(boolean isNewVariablesPresent){ Map externalVariablesMap = new LinkedHashMap<>(); if(isNewVariablesPresent) { - collectedVariablesMap.put("VAR1", null); - externalVariablesMap.put("EXTVAR1", null); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, null); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, null); } LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( @@ -645,17 +533,7 @@ void shouldKeepNull(boolean isNewVariablesPresent){ ); //THEN - assertThat(surveyUnitModels).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .varId()).isEqualTo("VAR1"); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .value()).isNull(); - assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .varId()).isEqualTo("EXTVAR1"); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .value()).isNull(); + assertNullVariables(surveyUnitModels); } @ParameterizedTest @@ -666,26 +544,10 @@ void shouldKeepNullIteration(boolean isNewVariablesPresent){ VariablesMap variablesMap = mock(VariablesMap.class); //Already existing survey unit - String interrogationId = "testInterrogation"; - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .collectionInstrumentId(QUESTIONNAIRE_ID) - .interrogationId(interrogationId) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); surveyUnitModel.getCollectedVariables().add( VariableModel.builder() - .varId("VAR1") - .value("1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getCollectedVariables().add( - VariableModel.builder() - .varId("VAR1") + .varId(COLLECTED_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -694,16 +556,7 @@ void shouldKeepNullIteration(boolean isNewVariablesPresent){ ); surveyUnitModel.getExternalVariables().add( VariableModel.builder() - .varId("EXTVAR1") - .value("ext1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getExternalVariables().add( - VariableModel.builder() - .varId("EXTVAR1") + .varId(EXTERNAL_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -716,27 +569,24 @@ void shouldKeepNullIteration(boolean isNewVariablesPresent){ surveyUnitModel )); - //Raw response + //Raw response with null or absent second iteration Map collectedVariablesMap = new LinkedHashMap<>(); Map externalVariablesMap = new LinkedHashMap<>(); + List collectedVariableValues = new ArrayList<>(); + collectedVariableValues.add(COLLECTED_VARIABLE_VALUE); + List externalVariableValues = new ArrayList<>(); + externalVariableValues.add(EXTERNAL_VARIABLE_VALUE); if(isNewVariablesPresent) { - List variablesStrings = new ArrayList<>(); - variablesStrings.add("VALUE1"); - variablesStrings.add(null); - collectedVariablesMap.put("VAR1", Map.of("COLLECTED", variablesStrings)); - - variablesStrings = new ArrayList<>(); - variablesStrings.add("ext1"); - variablesStrings.add(null); - externalVariablesMap.put("EXTVAR1", variablesStrings); - } else { - collectedVariablesMap.put("VAR1", Map.of("COLLECTED", List.of("VALUE1"))); - externalVariablesMap.put("EXTVAR1", List.of("ext1")); + collectedVariableValues.add(null); + externalVariableValues.add(null); } + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", collectedVariableValues)); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, externalVariableValues); + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( Map.of( "COLLECTED", collectedVariablesMap, @@ -754,18 +604,90 @@ void shouldKeepNullIteration(boolean isNewVariablesPresent){ ); //THEN + assertSecondIterationNull(surveyUnitModels); + } + + //NullVariablesTests UTILS + private SurveyUnitModel getSurveyUnitModel() { + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(QUESTIONNAIRE_ID) + .interrogationId(INTERROGATION_ID) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value(COLLECTED_VARIABLE_VALUE) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value(EXTERNAL_VARIABLE_VALUE) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + return surveyUnitModel; + } + + private void assertNullVariables(List surveyUnitModels) { assertThat(surveyUnitModels).hasSize(1); assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .varId()).isEqualTo("VAR1"); + .varId()).isEqualTo(COLLECTED_VARIABLE_NAME); assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() .value()).isNull(); assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .varId()).isEqualTo("EXTVAR1"); + .varId()).isEqualTo(EXTERNAL_VARIABLE_NAME); assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() .value()).isNull(); } + + private void assertNonNullVariables(List surveyUnitModels) { + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo(COLLECTED_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isEqualTo(COLLECTED_VARIABLE_VALUE); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo(EXTERNAL_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isEqualTo(EXTERNAL_VARIABLE_VALUE); + } + + private void assertSecondIterationNull(List surveyUnitModels) { + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getCollectedVariables()){ + assertThat(variableModel.iteration()).isIn(1,2); + if(variableModel.iteration().equals(2)){ + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo(COLLECTED_VARIABLE_VALUE); + } + + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getExternalVariables()) { + assertThat(variableModel.iteration()).isIn(1,2); + if (variableModel.iteration().equals(2)) { + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo(EXTERNAL_VARIABLE_VALUE); + } + } + } private LunaticJsonRawDataModel rawData(Map data) { diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java index 20406b4b4..7dcc15ab3 100644 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java @@ -297,6 +297,13 @@ void shouldConvertPairwiseCollectedVariables() { @DisplayName("Null cases tests") class NullVariablesTests { + //NullVariablesTests constants + private static final String INTERROGATION_ID = "testInterrogation"; + private static final String COLLECTED_VARIABLE_NAME = "VAR1"; + private static final String COLLECTED_VARIABLE_VALUE = "test"; + private static final String EXTERNAL_VARIABLE_NAME = "EXTVAR1"; + private static final String EXTERNAL_VARIABLE_VALUE = "ext1"; + @Test @DisplayName("Should not add variable if not already existant and null in raw") void shouldNotAddIfNull(){ @@ -304,32 +311,7 @@ void shouldNotAddIfNull(){ VariablesMap variablesMap = mock(VariablesMap.class); //Already existing survey unit - String interrogationId = "testInterrogation"; - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) - .interrogationId(interrogationId) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - - surveyUnitModel.getCollectedVariables().add( - VariableModel.builder() - .varId("VAR1") - .value("1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getExternalVariables().add( - VariableModel.builder() - .varId("EXTVAR1") - .value("ext1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) .thenReturn(List.of( @@ -338,11 +320,12 @@ void shouldNotAddIfNull(){ //Raw response with new null variables Map collectedVariablesMap = new LinkedHashMap<>(); - collectedVariablesMap.put("VAR1", Map.of("COLLECTED", "test")); + + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", COLLECTED_VARIABLE_VALUE)); collectedVariablesMap.put("VAR2", null); Map externalVariablesMap = new LinkedHashMap<>(); - externalVariablesMap.put("EXTVAR1", "ext1"); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, EXTERNAL_VARIABLE_VALUE); externalVariablesMap.put("EXTVAR2", null); RawResponseModel rawResponseModel = rawResponseModel( @@ -362,36 +345,30 @@ void shouldNotAddIfNull(){ ); //THEN - assertThat(surveyUnitModels).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .value()).isEqualTo("test"); - assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .value()).isEqualTo("ext1"); - + // No new variables added + assertNonNullVariables(surveyUnitModels); } @ParameterizedTest @ValueSource(booleans = {false, true}) @DisplayName("Should convert to non null if existant variable is null or absent") - void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPresent){ + void shouldConvertToNonNullValueIfNullOrNotExists(boolean addNullVariables){ //GIVEN VariablesMap variablesMap = mock(VariablesMap.class); //Already existing survey unit - String interrogationId = "testInterrogation"; SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) - .interrogationId(interrogationId) + .interrogationId(INTERROGATION_ID) .collectedVariables(new ArrayList<>()) .externalVariables(new ArrayList<>()) .build(); - - if(isVariableAlreadyPresent) { + + //Add null variables if true + if(addNullVariables) { surveyUnitModel.getCollectedVariables().add( VariableModel.builder() - .varId("VAR1") + .varId(COLLECTED_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -400,7 +377,7 @@ void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPrese ); surveyUnitModel.getExternalVariables().add( VariableModel.builder() - .varId("EXTVAR1") + .varId(EXTERNAL_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -416,10 +393,10 @@ void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPrese //Raw response Map collectedVariablesMap = new LinkedHashMap<>(); - collectedVariablesMap.put("VAR1", Map.of("COLLECTED", "1")); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", COLLECTED_VARIABLE_VALUE)); Map externalVariablesMap = new LinkedHashMap<>(); - externalVariablesMap.put("EXTVAR1", "ext1"); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, EXTERNAL_VARIABLE_VALUE); RawResponseModel rawResponseModel = rawResponseModel( payloadWith( @@ -438,17 +415,7 @@ void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPrese ); //THEN - assertThat(surveyUnitModels).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .varId()).isEqualTo("VAR1"); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .value()).isEqualTo("1"); - assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .varId()).isEqualTo("EXTVAR1"); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .value()).isEqualTo("ext1"); + assertNonNullVariables(surveyUnitModels); } @Test @@ -458,32 +425,7 @@ void shouldConvertNullValueIfNonNullOneValue(){ VariablesMap variablesMap = mock(VariablesMap.class); //Already existing survey unit with non-null variables - String interrogationId = "testInterrogation"; - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) - .interrogationId(interrogationId) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - - surveyUnitModel.getCollectedVariables().add( - VariableModel.builder() - .varId("VAR1") - .value("1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getExternalVariables().add( - VariableModel.builder() - .varId("EXTVAR1") - .value("ext1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) .thenReturn(List.of( @@ -494,8 +436,8 @@ void shouldConvertNullValueIfNonNullOneValue(){ Map collectedVariablesMap = new LinkedHashMap<>(); Map externalVariablesMap = new LinkedHashMap<>(); - collectedVariablesMap.put("VAR1", null); - externalVariablesMap.put("EXTVAR1", null); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, null); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, null); RawResponseModel rawResponseModel = rawResponseModel( @@ -515,12 +457,7 @@ void shouldConvertNullValueIfNonNullOneValue(){ ); //THEN - assertThat(surveyUnitModels).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst()).isNull(); - - assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst()).isNull(); + assertNullVariables(surveyUnitModels); } @Test @@ -530,26 +467,10 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ VariablesMap variablesMap = mock(VariablesMap.class); //Already existing survey unit with non-null variables - String interrogationId = "testInterrogation"; - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) - .interrogationId(interrogationId) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); surveyUnitModel.getCollectedVariables().add( VariableModel.builder() - .varId("VAR1") - .value("VALUE1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getCollectedVariables().add( - VariableModel.builder() - .varId("VAR1") + .varId(COLLECTED_VARIABLE_NAME) .value("VALUE2") .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -558,16 +479,7 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ ); surveyUnitModel.getExternalVariables().add( VariableModel.builder() - .varId("EXTVAR1") - .value("ext1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getExternalVariables().add( - VariableModel.builder() - .varId("EXTVAR1") + .varId(EXTERNAL_VARIABLE_NAME) .value("ext2") .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -585,14 +497,14 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ Map externalVariablesMap = new LinkedHashMap<>(); List variablesStrings = new ArrayList<>(); - variablesStrings.add("VALUE1"); + variablesStrings.add(COLLECTED_VARIABLE_VALUE); variablesStrings.add(null); - collectedVariablesMap.put("VAR1", Map.of("COLLECTED", variablesStrings)); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", variablesStrings)); variablesStrings = new ArrayList<>(); - variablesStrings.add("ext1"); + variablesStrings.add(EXTERNAL_VARIABLE_VALUE); variablesStrings.add(null); - externalVariablesMap.put("EXTVAR1", variablesStrings); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, variablesStrings); RawResponseModel rawResponseModel = rawResponseModel( @@ -612,26 +524,7 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ ); //THEN - assertThat(surveyUnitModels).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(2); - for(VariableModel variableModel : surveyUnitModels.getFirst().getCollectedVariables()){ - assertThat(variableModel.iteration()).isIn(0,1); - if(variableModel.iteration().equals(1)){ - assertThat(variableModel.value()).isNull(); - continue; - } - assertThat(variableModel.value()).isEqualTo("VALUE1"); - } - - assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(2); - for(VariableModel variableModel : surveyUnitModels.getFirst().getExternalVariables()) { - assertThat(variableModel.iteration()).isIn(0, 1); - if (variableModel.iteration().equals(1)) { - assertThat(variableModel.value()).isNull(); - continue; - } - assertThat(variableModel.value()).isEqualTo("ext1"); - } + assertSecondIterationNull(surveyUnitModels); } @ParameterizedTest @@ -641,18 +534,16 @@ void shouldKeepNull(boolean isNewVariablesPresent){ //GIVEN VariablesMap variablesMap = mock(VariablesMap.class); - //Already existing survey unit - String interrogationId = "testInterrogation"; + //Already existing survey unit with null variables SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) - .interrogationId(interrogationId) + .interrogationId(INTERROGATION_ID) .collectedVariables(new ArrayList<>()) .externalVariables(new ArrayList<>()) .build(); - surveyUnitModel.getCollectedVariables().add( VariableModel.builder() - .varId("VAR1") + .varId(COLLECTED_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -661,7 +552,7 @@ void shouldKeepNull(boolean isNewVariablesPresent){ ); surveyUnitModel.getExternalVariables().add( VariableModel.builder() - .varId("EXTVAR1") + .varId(EXTERNAL_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -674,13 +565,12 @@ void shouldKeepNull(boolean isNewVariablesPresent){ surveyUnitModel )); - //Raw response + //Raw response with null variables or no variable at all Map collectedVariablesMap = new LinkedHashMap<>(); Map externalVariablesMap = new LinkedHashMap<>(); - if(isNewVariablesPresent) { - collectedVariablesMap.put("VAR1", null); - externalVariablesMap.put("EXTVAR1", null); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, null); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, null); } RawResponseModel rawResponseModel = rawResponseModel( @@ -700,47 +590,21 @@ void shouldKeepNull(boolean isNewVariablesPresent){ ); //THEN - assertThat(surveyUnitModels).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .varId()).isEqualTo("VAR1"); - assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .value()).isNull(); - assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .varId()).isEqualTo("EXTVAR1"); - assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .value()).isNull(); + assertNullVariables(surveyUnitModels); } @ParameterizedTest @ValueSource(booleans = {false, true}) @DisplayName("Should keep null value if variable null or absent (multiple iterations)") - void shouldKeepNullIteration(boolean isNewVariablesPresent){ + void shouldKeepNullIteration(boolean isSecondIterationPresent){ //GIVEN VariablesMap variablesMap = mock(VariablesMap.class); - //Already existing survey unit - String interrogationId = "testInterrogation"; - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) - .interrogationId(interrogationId) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - + //Already existing survey unit with null second iterations + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); surveyUnitModel.getCollectedVariables().add( VariableModel.builder() - .varId("VAR1") - .value("1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getCollectedVariables().add( - VariableModel.builder() - .varId("VAR1") + .varId(COLLECTED_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -749,16 +613,7 @@ void shouldKeepNullIteration(boolean isNewVariablesPresent){ ); surveyUnitModel.getExternalVariables().add( VariableModel.builder() - .varId("EXTVAR1") - .value("ext1") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build() - ); - surveyUnitModel.getExternalVariables().add( - VariableModel.builder() - .varId("EXTVAR1") + .varId(EXTERNAL_VARIABLE_NAME) .value(null) .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) @@ -771,27 +626,24 @@ void shouldKeepNullIteration(boolean isNewVariablesPresent){ surveyUnitModel )); - //Raw response + //Raw response with null second iteration or no second iteration at all Map collectedVariablesMap = new LinkedHashMap<>(); Map externalVariablesMap = new LinkedHashMap<>(); + List collectedVariablesValues = new ArrayList<>(); + collectedVariablesValues.add(COLLECTED_VARIABLE_VALUE); + List externalVariablesValues = new ArrayList<>(); + externalVariablesValues.add(EXTERNAL_VARIABLE_VALUE); - if(isNewVariablesPresent) { - List variablesStrings = new ArrayList<>(); - variablesStrings.add("VALUE1"); - variablesStrings.add(null); - collectedVariablesMap.put("VAR1", Map.of("COLLECTED", variablesStrings)); - - variablesStrings = new ArrayList<>(); - variablesStrings.add("ext1"); - variablesStrings.add(null); - externalVariablesMap.put("EXTVAR1", variablesStrings); - } else { - collectedVariablesMap.put("VAR1", Map.of("COLLECTED", List.of("VALUE1"))); - externalVariablesMap.put("EXTVAR1", List.of("ext1")); + if(isSecondIterationPresent) { + collectedVariablesValues.add(null); + externalVariablesValues.add(null); } + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", collectedVariablesValues)); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, externalVariablesValues); + RawResponseModel rawResponseModel = rawResponseModel( payloadWith( Map.of("COLLECTED", collectedVariablesMap), @@ -809,18 +661,90 @@ void shouldKeepNullIteration(boolean isNewVariablesPresent){ ); //THEN + assertSecondIterationNull(surveyUnitModels); + } + + //NullVariablesTests UTILS + private SurveyUnitModel getSurveyUnitModel() { + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .interrogationId(INTERROGATION_ID) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value(COLLECTED_VARIABLE_VALUE) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value(EXTERNAL_VARIABLE_VALUE) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + return surveyUnitModel; + } + + private void assertNullVariables(List surveyUnitModels) { assertThat(surveyUnitModels).hasSize(1); assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() - .varId()).isEqualTo("VAR1"); + .varId()).isEqualTo(NullVariablesTests.COLLECTED_VARIABLE_NAME); assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() .value()).isNull(); assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() - .varId()).isEqualTo("EXTVAR1"); + .varId()).isEqualTo(NullVariablesTests.EXTERNAL_VARIABLE_NAME); assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() .value()).isNull(); } + + private void assertNonNullVariables(List surveyUnitModels) { + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo(COLLECTED_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isEqualTo(COLLECTED_VARIABLE_VALUE); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo(EXTERNAL_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isEqualTo(EXTERNAL_VARIABLE_VALUE); + } + + private void assertSecondIterationNull(List surveyUnitModels) { + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getCollectedVariables()){ + assertThat(variableModel.iteration()).isIn(1,2); + if(variableModel.iteration().equals(2)){ + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo(COLLECTED_VARIABLE_VALUE); + } + + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getExternalVariables()) { + assertThat(variableModel.iteration()).isIn(1,2); + if (variableModel.iteration().equals(2)) { + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo(EXTERNAL_VARIABLE_VALUE); + } + } + } From 5d7ba86e4de867ac287d7a52d95baac62ec85b31 Mon Sep 17 00:00:00 2001 From: Alexis Szmundy Date: Thu, 4 Jun 2026 15:03:35 +0200 Subject: [PATCH 04/12] test: null raw process IT --- .../responses/RawResponseControllerIT.java | 409 ++++++++++++++++-- 1 file changed, 384 insertions(+), 25 deletions(-) diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java index db58fdfb3..3e5c62e68 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java @@ -231,7 +231,7 @@ void saveLunaticJson_syntax_or_json_schema_error_test(String jsonFilePathString) @Nested @DisplayName("Filiere model raw data processing tests") - class rawDataProcessingTests { + class RawResponsesProcessingTests { //HAPPY PATHS @Test @WithMockUser(roles = "SCHEDULER") @@ -258,7 +258,8 @@ void process_raw_response_interrogation_list_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -329,7 +330,8 @@ void process_raw_response_interrogation_list_no_collected_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -397,7 +399,8 @@ void process_raw_response_collectionInstrumentId_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -443,6 +446,147 @@ void process_raw_response_collectionInstrumentId_test(){ } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @WithMockUser(roles = "SCHEDULER") + @DisplayName("Filiere model raw data null values should be kept if null or non null value already exists") + @SneakyThrows + void process_raw_response_keep_null_test(boolean isNullSurveyUnitValues) { + //GIVEN + String collectionInstrumentId = "TESTQUEST"; + Mode mode = Mode.WEB; + List interrogationIds = List.of("INTERRO1"); + + //Raw responses with null variables + String variableName = "VAR1"; + String collectedValue = "value1"; + Map collectedVariablesAndValues = new HashMap<>(); + collectedVariablesAndValues.put(variableName, collectedValue); + + String externalVariableName = "EXTVAR1"; + String externalValue = "externalvalue1"; + Map externalVariablesAndValues = new HashMap<>(); + externalVariablesAndValues.put(externalVariableName, externalValue); + + setFiliereModelTestMockBehaviour( + collectionInstrumentId, + mode, + interrogationIds, + collectedVariablesAndValues, + externalVariablesAndValues, + true + ); + + //Survey unit that already exists for first interrogation + SurveyUnitDocument alreadyPresentSurveyUnitDocument = getSurveyUnitDocument( + collectionInstrumentId, + interrogationIds.getFirst(), + variableName, + isNullSurveyUnitValues ? null : collectedValue, + externalVariableName, + isNullSurveyUnitValues ? null : externalValue + ); + + when(surveyUnitMongoDBRepository.findByCollectionInstrumentIdAndInterrogationIds(collectionInstrumentId, interrogationIds)) + .thenReturn(List.of(alreadyPresentSurveyUnitDocument)); + + // WHEN + mockMvc.perform(post("/raw-responses/%s/process".formatted(collectionInstrumentId)) + .with(csrf()) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + + //THEN + @SuppressWarnings("unchecked") + ArgumentCaptor> listArgumentCaptor = + ArgumentCaptor.forClass(List.class); + verify(surveyUnitMongoDBRepository, times(1)) + .insert(listArgumentCaptor.capture()); + + //Document must have null variables values + List savedDocuments = listArgumentCaptor.getValue(); + Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); + SurveyUnitDocument savedDocument = savedDocuments.stream().filter( + surveyUnitDocument -> + surveyUnitDocument.getInterrogationId().equals("interrogationId")) + .toList().getFirst(); + Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); + VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); + Assertions.assertThat(variableDocument.getValue()).isNull(); + Assertions.assertThat(savedDocument.getExternalVariables()).isNotNull().hasSize(1); + variableDocument = savedDocument.getExternalVariables().getFirst(); + Assertions.assertThat(variableDocument.getValue()).isNull(); + } + + @Test + @WithMockUser(roles = "SCHEDULER") + @DisplayName("Filiere model raw data shouldn't overwrite with null if not existent before") + @SneakyThrows + void process_raw_response_null_on_absent_test() { + //GIVEN + String collectionInstrumentId = "TESTQUEST"; + Mode mode = Mode.WEB; + List interrogationIds = List.of("INTERRO1"); + + //Raw responses with null variables + String variableName = "VAR1"; + Map collectedVariablesAndValues = new HashMap<>(); + collectedVariablesAndValues.put(variableName, null); + + String externalVariableName = "EXTVAR1"; + Map externalVariablesAndValues = new HashMap<>(); + externalVariablesAndValues.put(externalVariableName, null); + + setFiliereModelTestMockBehaviour( + collectionInstrumentId, + mode, + interrogationIds, + collectedVariablesAndValues, + externalVariablesAndValues, + true + ); + + //Survey unit that already exists for first interrogation with variables not present in raw + SurveyUnitDocument alreadyPresentSurveyUnitDocument = getSurveyUnitDocument( + collectionInstrumentId, + interrogationIds.getFirst(), + "VAR2", + "value", + "EXTVAR2", + "externalValue" + ); + + when(surveyUnitMongoDBRepository.findByCollectionInstrumentIdAndInterrogationIds(collectionInstrumentId, interrogationIds)) + .thenReturn(List.of(alreadyPresentSurveyUnitDocument)); + + // WHEN + mockMvc.perform(post("/raw-responses/%s/process".formatted(collectionInstrumentId)) + .with(csrf()) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + + //THEN + @SuppressWarnings("unchecked") + ArgumentCaptor> listArgumentCaptor = + ArgumentCaptor.forClass(List.class); + verify(surveyUnitMongoDBRepository, times(1)) + .insert(listArgumentCaptor.capture()); + + //Document shouldn't have null variables + List savedDocuments = listArgumentCaptor.getValue(); + Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); + SurveyUnitDocument savedDocument = savedDocuments.stream().filter( + surveyUnitDocument -> + surveyUnitDocument.getInterrogationId().equals("interrogationId")) + .toList().getFirst(); + Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); + VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); + Assertions.assertThat(variableDocument.getVarId()).isNotEqualTo(variableName); + Assertions.assertThat(savedDocument.getExternalVariables()).isNotNull().hasSize(1); + variableDocument = savedDocument.getExternalVariables().getFirst(); + Assertions.assertThat(variableDocument.getVarId()).isNotEqualTo(externalVariableName); + } + @Test @WithMockUser(roles = "SCHEDULER") @DisplayName("Filiere model raw data should have processed date if no data") @@ -463,7 +607,8 @@ void process_raw_response_noData_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -496,7 +641,8 @@ private void setFiliereModelTestMockBehaviour( Mode mode, List interrogationIds, Map collectedVariableNamesAndValues, - Map externalVariableNamesAndValues + Map externalVariableNamesAndValues, + boolean isNullValues ){ //Disable withReview DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument(); @@ -545,6 +691,29 @@ private void setFiliereModelTestMockBehaviour( ).thenReturn(interrogationIds); //FILIERE RAW DOCS + List rawResponseDocuments = getRawResponseDocuments( + collectionInstrumentId, + mode, + interrogationIds, + collectedVariableNamesAndValues, + externalVariableNamesAndValues, + isNullValues + ); + when(rawResponseRepository.findByCollectionInstrumentIdAndModeAndInterrogationIdList( + eq(collectionInstrumentId), + eq(mode.getJsonName()), + argThat(argument -> argument.containsAll(interrogationIds)) //Any order + )).thenReturn(rawResponseDocuments); + } + + private List getRawResponseDocuments( + String collectionInstrumentId, + Mode mode, + List interrogationIds, + Map collectedVariableNamesAndValues, + Map externalVariableNamesAndValues, + boolean isNullValues + ) { List rawResponseDocuments = new ArrayList<>(); for(String interrogationId : interrogationIds){ Map dataMap = getNewDataMap(); @@ -552,14 +721,14 @@ private void setFiliereModelTestMockBehaviour( for(Map.Entry variable : collectedVariableNamesAndValues.entrySet()) { String variableName = variable.getKey(); //To have different value on each interrogation - String value = interrogationId + variable.getValue(); + String value = isNullValues ? null : interrogationId + variable.getValue(); addVariableToDataMap(dataMap, variableName, value, Constants.COLLECTED_NODE_NAME); } //EXTERNAL VARIABLES for(Map.Entry variable : externalVariableNamesAndValues.entrySet()) { String variableName = variable.getKey(); - String value = interrogationId + variable.getValue(); + String value = isNullValues ? null : interrogationId + variable.getValue(); addVariableToDataMap(dataMap, variableName, value, Constants.EXTERNAL_NODE_NAME); } @@ -581,17 +750,13 @@ private void setFiliereModelTestMockBehaviour( .build(); rawResponseDocuments.add(rawResponseDocument); } - when(rawResponseRepository.findByCollectionInstrumentIdAndModeAndInterrogationIdList( - eq(collectionInstrumentId), - eq(mode.getJsonName()), - argThat(argument -> argument.containsAll(interrogationIds)) //Any order - )).thenReturn(rawResponseDocuments); + return rawResponseDocuments; } } @Nested @DisplayName("Old model raw data processing tests") - class lunaticJsonDataProcessingTests { + class LunaticJsonDataProcessingTests { @Test @WithMockUser(roles = "SCHEDULER") @DisplayName("Old model raw data should be processed using the questionnaireId") @@ -618,7 +783,8 @@ void processLunaticJsonRawData_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -664,6 +830,147 @@ void processLunaticJsonRawData_test(){ } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @WithMockUser(roles = "SCHEDULER") + @DisplayName("Old model raw data null values should be kept if non null value already exists") + @SneakyThrows + void processLunaticJsonRawData_keep_null_test(boolean isNullSurveyUnitValues){ + //GIVEN + String questionnaireId = "TESTQUEST"; + Mode mode = Mode.WEB; + + List interrogationIds = List.of("INTERRO1"); + + String variableName = "VAR1"; + String collectedValue = "value1"; + Map collectedVariablesAndValues = new HashMap<>(); + collectedVariablesAndValues.put(variableName, collectedValue); + + String externalVariableName = "EXTVAR1"; + String externalValue = "externalvalue1"; + Map externalVariablesAndValues = new HashMap<>(); + externalVariablesAndValues.put(externalVariableName, externalValue); + + setOldModelTestMockBehaviour( + questionnaireId, + mode, + interrogationIds, + collectedVariablesAndValues, + externalVariablesAndValues, + true + ); + + //Survey unit that already exists for first interrogation + SurveyUnitDocument alreadyPresentSurveyUnitDocument = getSurveyUnitDocument( + questionnaireId, + interrogationIds.getFirst(), + variableName, + isNullSurveyUnitValues ? null : collectedValue, + externalVariableName, + isNullSurveyUnitValues ? null : externalValue + ); + when(surveyUnitMongoDBRepository.findByCollectionInstrumentIdAndInterrogationIds(questionnaireId, interrogationIds)) + .thenReturn(List.of(alreadyPresentSurveyUnitDocument)); + + // WHEN + mockMvc.perform(post("/responses/raw/lunatic-json/%s/process".formatted(questionnaireId)) + .with(csrf()) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + + //THEN + @SuppressWarnings("unchecked") + ArgumentCaptor> listArgumentCaptor = + ArgumentCaptor.forClass(List.class); + verify(surveyUnitMongoDBRepository, times(1)) + .insert(listArgumentCaptor.capture()); + + //Document must have null variables values + List savedDocuments = listArgumentCaptor.getValue(); + Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); + SurveyUnitDocument savedDocument = savedDocuments.stream().filter( + surveyUnitDocument -> + surveyUnitDocument.getInterrogationId().equals("interrogationId")) + .toList().getFirst(); + Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); + VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); + Assertions.assertThat(variableDocument.getValue()).isNull(); + Assertions.assertThat(savedDocument.getExternalVariables()).isNotNull().hasSize(1); + variableDocument = savedDocument.getExternalVariables().getFirst(); + Assertions.assertThat(variableDocument.getValue()).isNull(); + } + + @Test + @WithMockUser(roles = "SCHEDULER") + @DisplayName("Old model raw data null values should be kept if non null value already exists") + @SneakyThrows + void processLunaticJsonRawData_null_on_absent_test(){ + //GIVEN + String questionnaireId = "TESTQUEST"; + Mode mode = Mode.WEB; + + List interrogationIds = List.of("INTERRO1"); + + String variableName = "VAR1"; + String collectedValue = "value1"; + Map collectedVariablesAndValues = new HashMap<>(); + collectedVariablesAndValues.put(variableName, collectedValue); + + String externalVariableName = "EXTVAR1"; + String externalValue = "externalvalue1"; + Map externalVariablesAndValues = new HashMap<>(); + externalVariablesAndValues.put(externalVariableName, externalValue); + + setOldModelTestMockBehaviour( + questionnaireId, + mode, + interrogationIds, + collectedVariablesAndValues, + externalVariablesAndValues, + true + ); + + //Survey unit that already exists for first interrogation + SurveyUnitDocument alreadyPresentSurveyUnitDocument = getSurveyUnitDocument( + questionnaireId, + interrogationIds.getFirst(), + "VAR2", + "value", + "EXTVAR2", + "externalValue" + ); + when(surveyUnitMongoDBRepository.findByCollectionInstrumentIdAndInterrogationIds(questionnaireId, interrogationIds)) + .thenReturn(List.of(alreadyPresentSurveyUnitDocument)); + + // WHEN + mockMvc.perform(post("/responses/raw/lunatic-json/%s/process".formatted(questionnaireId)) + .with(csrf()) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + + //THEN + @SuppressWarnings("unchecked") + ArgumentCaptor> listArgumentCaptor = + ArgumentCaptor.forClass(List.class); + verify(surveyUnitMongoDBRepository, times(1)) + .insert(listArgumentCaptor.capture()); + + //Document shouldn't have null variables + List savedDocuments = listArgumentCaptor.getValue(); + Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); + SurveyUnitDocument savedDocument = savedDocuments.stream().filter( + surveyUnitDocument -> + surveyUnitDocument.getInterrogationId().equals("interrogationId")) + .toList().getFirst(); + Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); + VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); + Assertions.assertThat(variableDocument.getVarId()).isNotEqualTo(variableName); + Assertions.assertThat(savedDocument.getExternalVariables()).isNotNull().hasSize(1); + variableDocument = savedDocument.getExternalVariables().getFirst(); + Assertions.assertThat(variableDocument.getVarId()).isNotEqualTo(externalVariableName); + } + @Test @WithMockUser(roles = "SCHEDULER") @DisplayName("Old model raw data should be processed even without collected") @@ -688,7 +995,8 @@ void processLunaticJsonRawData_no_collected_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -754,7 +1062,8 @@ void processLunaticJsonRawData_noData_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -791,7 +1100,8 @@ private void setOldModelTestMockBehaviour(String questionnaireId, Mode mode, List interrogationIds, Map collectedVariableNamesAndValues, - Map externalVariableNamesAndValues + Map externalVariableNamesAndValues, + boolean isNullValues ){ //Disable withReview DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument(); @@ -845,6 +1155,29 @@ private void setOldModelTestMockBehaviour(String questionnaireId, .thenReturn(List.of(groupedInterrogationDocument)); //LUNATIC JSON RAW DOCS + List lunaticJsonRawDataDocuments = getLunaticJsonRawDataDocuments( + questionnaireId, + mode, + interrogationIds, + collectedVariableNamesAndValues, + externalVariableNamesAndValues, + isNullValues + ); + when(lunaticJsonMongoDBRepository.findByQuestionnaireModeAndInterrogations( + eq(questionnaireId), + eq(mode), + anyList() + )).thenReturn(lunaticJsonRawDataDocuments); + } + + private List getLunaticJsonRawDataDocuments( + String questionnaireId, + Mode mode, + List interrogationIds, + Map collectedVariableNamesAndValues, + Map externalVariableNamesAndValues, + boolean isNullValues + ) { List lunaticJsonRawDataDocuments = new ArrayList<>(); for(String interrogationId : interrogationIds){ Map dataMap = getNewDataMap(); @@ -852,14 +1185,14 @@ private void setOldModelTestMockBehaviour(String questionnaireId, for(Map.Entry variable : collectedVariableNamesAndValues.entrySet()) { String variableName = variable.getKey(); //To have different value on each interrogation - String value = interrogationId + variable.getValue(); + String value = isNullValues ? null : interrogationId + variable.getValue(); addVariableToDataMap(dataMap, variableName, value, Constants.COLLECTED_NODE_NAME); } //EXTERNAL VARIABLES for(Map.Entry variable : externalVariableNamesAndValues.entrySet()) { String variableName = variable.getKey(); - String value = interrogationId + variable.getValue(); + String value = isNullValues ? null : interrogationId + variable.getValue(); addVariableToDataMap(dataMap, variableName, value, Constants.EXTERNAL_NODE_NAME); } @@ -875,11 +1208,7 @@ private void setOldModelTestMockBehaviour(String questionnaireId, .build(); lunaticJsonRawDataDocuments.add(lunaticJsonRawDataDocument); } - when(lunaticJsonMongoDBRepository.findByQuestionnaireModeAndInterrogations( - eq(questionnaireId), - eq(mode), - anyList() - )).thenReturn(lunaticJsonRawDataDocuments); + return lunaticJsonRawDataDocuments; } } @@ -928,5 +1257,35 @@ private void addExternalVariableToDataMap(Map dataMap, externalMap.put(variableName, value); } + private SurveyUnitDocument getSurveyUnitDocument( + String collectionInstrumentId, + String interrogationId, + String variableName, + String collectedValue, + String externalVariableName, + String externalValue + ) { + SurveyUnitDocument alreadyPresentSurveyUnitDocument = new SurveyUnitDocument(); + alreadyPresentSurveyUnitDocument.setCollectionInstrumentId(collectionInstrumentId); + alreadyPresentSurveyUnitDocument.setInterrogationId(interrogationId); + alreadyPresentSurveyUnitDocument.setMode(Mode.WEB.getModeName()); + + alreadyPresentSurveyUnitDocument.setCollectedVariables(new ArrayList<>()); + VariableDocument oldVariable = new VariableDocument(); + oldVariable.setVarId(variableName); + oldVariable.setIteration(1); + oldVariable.setValue(collectedValue); + oldVariable.setScope(Constants.ROOT_GROUP_NAME); + alreadyPresentSurveyUnitDocument.getCollectedVariables().add(oldVariable); + + alreadyPresentSurveyUnitDocument.setExternalVariables(new ArrayList<>()); + oldVariable = new VariableDocument(); + oldVariable.setVarId(externalVariableName); + oldVariable.setIteration(1); + oldVariable.setValue(externalValue); + oldVariable.setScope(Constants.ROOT_GROUP_NAME); + return alreadyPresentSurveyUnitDocument; + } + //TODO GET tests } \ No newline at end of file From 7d19d3c5a4b1e4822c2b0f4408742f429b91e54e Mon Sep 17 00:00:00 2001 From: Alexis Szmundy Date: Thu, 4 Jun 2026 15:33:14 +0200 Subject: [PATCH 05/12] test: add get latest null IT --- .../rawdata/LunaticJsonRawDataConverter.java | 3 +- .../rawdata/RawResponseRawDataConverter.java | 15 ++-- .../rest/responses/ResponseControllerIT.java | 75 +++++++++++++++++++ .../LunaticJsonRawDataConverterTest.java | 2 +- 4 files changed, 87 insertions(+), 8 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java index cd11607cf..760ea93b4 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java @@ -12,7 +12,6 @@ import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.GroupUtils; import fr.insee.genesis.domain.utils.JsonUtils; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; @@ -142,7 +141,7 @@ private void convertRawDataCollectedVariables( List destination = dstSurveyUnitModel.getCollectedVariables(); for (Map.Entry collectedVariable : collectedMap.entrySet()) { - RawResponseRawDataConverter.processCollectedVariable( + RawResponseRawDataConverter.processCollectedVariableForState( collectedVariable, stateKey, variablesMap, diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java index 59aaf91a7..72904b0df 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java @@ -161,7 +161,7 @@ private void convertCollectedVariables( List collectedVariables = dstSurveyUnitModel.getCollectedVariables(); for (Map.Entry entry : collectedMap.entrySet()) { - processCollectedVariable( + processCollectedVariableForState( entry, stateKey, variablesMap, @@ -172,7 +172,7 @@ private void convertCollectedVariables( } } - public static void processCollectedVariable( + public static void processCollectedVariableForState( Map.Entry entry, String stateKey, VariablesMap variablesMap, @@ -185,13 +185,14 @@ public static void processCollectedVariable( return; } - Map states = JsonUtils.asMap(entry.getValue()); - if (states == null || states.get(stateKey) == null) { + Map variableStates = JsonUtils.asMap(entry.getValue()); + //If variable value absent for state or null value + if (variableStates == null || variableStates.get(stateKey) == null) { convertNullVar(entry, lastSurveyUnitModel, variableModelList); return; } - Object value = states.get(stateKey); + Object value = variableStates.get(stateKey); if (value instanceof List list) { convertListVar(list, entry, variablesMap, variableModelList); return; @@ -204,7 +205,11 @@ private static void convertNullVar( @Nullable SurveyUnitModel lastSurveyUnitModel, List destination ) { + if(lastSurveyUnitModel == null){ + return; + } //TODO + } private static void convertListVar( diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerIT.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerIT.java index ac1b75ec2..7e9978a72 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerIT.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerIT.java @@ -117,6 +117,81 @@ void get_simplified_response_test() { } + @Test + @WithMockUser(roles = "USER_KRAFTWERK") + @DisplayName("Get simplified response with null variable") + @SneakyThrows + void get_simplified_response_null_variable_test() { + //GIVEN + String collectionInstrumentId = "collectionInstrumentId"; + String interrogationId = "interrogationId"; + String usualSurveyUnitId = "usualSurveyUnitId"; + Mode mode = Mode.WEB; + RawResponseDto.QuestionnaireStateEnum questionnaireStateEnum = + RawResponseDto.QuestionnaireStateEnum.FINISHED; + LocalDateTime validationDate = LocalDateTime.now().minusDays(2); + DataState dataState = DataState.COLLECTED; + + SurveyUnitDocument surveyUnitDocument = new SurveyUnitDocument(); + surveyUnitDocument.setCollectionInstrumentId(collectionInstrumentId); + surveyUnitDocument.setInterrogationId(interrogationId); + surveyUnitDocument.setMode(mode.getModeName()); + surveyUnitDocument.setState(dataState.name()); + surveyUnitDocument.setUsualSurveyUnitId(usualSurveyUnitId); + surveyUnitDocument.setQuestionnaireState(questionnaireStateEnum); + surveyUnitDocument.setValidationDate(validationDate); + surveyUnitDocument.setCollectedVariables(new ArrayList<>()); + surveyUnitDocument.setExternalVariables(new ArrayList<>()); + surveyUnitDocument.setRecordDate(Instant.now().minusSeconds(60)); + + String collectedVariableName = "var1"; + VariableDocument collectedVariableDocument = new VariableDocument(); + collectedVariableDocument.setVarId(collectedVariableName); + collectedVariableDocument.setValue(null); //null value + collectedVariableDocument.setIteration(1); + collectedVariableDocument.setScope(Constants.ROOT_GROUP_NAME); + surveyUnitDocument.getCollectedVariables().add(collectedVariableDocument); + + String externalVariableName = "extvar1"; + VariableDocument externalVariableDocument = new VariableDocument(); + externalVariableDocument.setVarId(externalVariableName); + externalVariableDocument.setValue(null); //null value + externalVariableDocument.setIteration(1); + externalVariableDocument.setScope(Constants.ROOT_GROUP_NAME); + surveyUnitDocument.getExternalVariables().add(externalVariableDocument); + + + Mockito.when(surveyUnitMongoDBRepository.findByInterrogationIdAndCollectionInstrumentId( + interrogationId, collectionInstrumentId + )).thenReturn(List.of(surveyUnitDocument)); + + //WHEN + THEN + mockMvc.perform(get("/responses/%s/%s/%s" + .formatted(collectionInstrumentId, mode.getModeName(), interrogationId) + ) + .with(csrf())) + .andExpect(status().isOk()) + + .andExpect(jsonPath("$.collectionInstrumentId").value(collectionInstrumentId)) + .andExpect(jsonPath("$.interrogationId").value(interrogationId)) + .andExpect(jsonPath("$.usualSurveyUnitId").value(usualSurveyUnitId)) + .andExpect(jsonPath("$.mode").value(mode.getModeName())) + .andExpect(jsonPath("$.validationDate").value(validationDate.format(DateTimeFormatter.ISO_DATE_TIME))) + .andExpect(jsonPath("$.questionnaireState").value(questionnaireStateEnum.name())) + + .andExpect(jsonPath("$.variablesUpdate[0].varId").value(collectedVariableName)) + .andExpect(jsonPath("$.variablesUpdate[0].value").isEmpty()) + .andExpect(jsonPath("$.variablesUpdate[0].state").value(dataState.name())) + .andExpect(jsonPath("$.variablesUpdate[0].scope").value(Constants.ROOT_GROUP_NAME)) + .andExpect(jsonPath("$.variablesUpdate[0].iteration").value(1)) + + .andExpect(jsonPath("$.externalVariables[0].varId").value(externalVariableName)) + .andExpect(jsonPath("$.externalVariables[0].value").isEmpty()) + .andExpect(jsonPath("$.externalVariables[0].state").value(dataState.name())) + .andExpect(jsonPath("$.externalVariables[0].scope").value(Constants.ROOT_GROUP_NAME)) + .andExpect(jsonPath("$.externalVariables[0].iteration").value(1)); + } + @Test @WithMockUser(roles = "USER_KRAFTWERK") @DisplayName("Get simplified response with an additionnal EDITED document") diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java index feee30065..5bd265982 100644 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java @@ -78,7 +78,7 @@ void shouldCreateCollectedAndEditedSurveyUnitsFromLegacyRawData() { try (MockedStatic rawResponseConverter = mockStatic(RawResponseRawDataConverter.class)) { rawResponseConverter - .when(() -> RawResponseRawDataConverter.processCollectedVariable(any(), any(), any(), any(), any(), any())) + .when(() -> RawResponseRawDataConverter.processCollectedVariableForState(any(), any(), any(), any(), any(), any())) .thenAnswer(invocation -> { String state = invocation.getArgument(1); List destination = invocation.getArgument(4); From 3529cb9cb594d331c50754cb10ddaf5ae573b537 Mon Sep 17 00:00:00 2001 From: Alexis Szmundy Date: Fri, 5 Jun 2026 17:16:19 +0200 Subject: [PATCH 06/12] feat(rawresponses): keep null values if variable present in last survey unit --- .../rawdata/LunaticJsonRawDataConverter.java | 18 +- .../converter/rawdata/RawDataConverter.java | 58 +++- .../rawdata/RawResponseRawDataConverter.java | 303 ++++++++++++++++-- .../rawdata/RawDataConverterTest.java | 51 +-- .../RawResponseRawDataConverterTest.java | 72 ++--- 5 files changed, 389 insertions(+), 113 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java index 760ea93b4..82201fc9f 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java @@ -53,14 +53,22 @@ public List convertRawDataAndCollectEmptyModels( List emptySurveyUnitModels ) { List surveyUnitModels = new ArrayList<>(); - Map lastSurveyUnitModelsByInterrogationId = getLastSurveyUnitModels( - questionnaireId, - rawDataList.stream().map(LunaticJsonRawDataModel::interrogationId).collect(Collectors.toList()) - ); + + Map> lastSurveyUnitModelsByInterrogationIdAndState = + getLastSurveyUnitModels( + questionnaireId, + rawDataList.stream().map(LunaticJsonRawDataModel::interrogationId).collect(Collectors.toList()) + ); for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) { for (LunaticJsonRawDataModel rawData : rawDataList) { RawDataModelType rawDataModelType = getRawDataModelType(rawData); + SurveyUnitModel lastSurveyUnitModelForDataState = null; + if(lastSurveyUnitModelsByInterrogationIdAndState.containsKey(rawData.interrogationId())){ + lastSurveyUnitModelForDataState = lastSurveyUnitModelsByInterrogationIdAndState + .get(rawData.interrogationId()) + .get(dataState); + } SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .collectionInstrumentId(rawData.questionnaireId()) @@ -79,7 +87,7 @@ public List convertRawDataAndCollectEmptyModels( convertRawDataCollectedVariables( rawData, - lastSurveyUnitModelsByInterrogationId.get(rawData.interrogationId()), + lastSurveyUnitModelForDataState, surveyUnitModel, dataState, rawDataModelType, diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java index d0db6951f..170759721 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java @@ -1,45 +1,75 @@ package fr.insee.genesis.domain.converter.rawdata; +import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; -import lombok.AllArgsConstructor; -import lombok.NoArgsConstructor; -import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; @Component -@NoArgsConstructor(force = true) -@RequiredArgsConstructor public abstract class RawDataConverter { + private SurveyUnitService surveyUnitService; @Autowired - private final SurveyUnitService surveyUnitService; - + public RawDataConverter(SurveyUnitService surveyUnitService) { + this.surveyUnitService = surveyUnitService; + } /** * @param questionnaireOrCollectionInstrumentId Questionnaire/Collection instrument id * @param interrogationIds list of interrogation ids * @return a Map containing latest survey unit models for each interrogation ids */ - protected Map getLastSurveyUnitModels( + protected Map> getLastSurveyUnitModels( String questionnaireOrCollectionInstrumentId, List interrogationIds ) { Set interrogationIdsSet = new HashSet<>(interrogationIds); - return surveyUnitService.findLatestByInterrogationIds( + List surveyUnitModels = surveyUnitService.findLatestByInterrogationIds( questionnaireOrCollectionInstrumentId, interrogationIdsSet - ).stream().collect(Collectors.toMap( - SurveyUnitModel::getInterrogationId, - surveyUnitModel -> surveyUnitModel - )); + ); + + Map> surveyUnitModelsByInterrogationIdAndState = new HashMap<>(); + + for (String interrogationId : interrogationIdsSet){ + List surveyUnitModelsForInterrogationId = surveyUnitModels.stream().filter( + surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(interrogationId) + ).toList(); + if(surveyUnitModelsForInterrogationId.isEmpty()){ + continue; + } + addSurveyUnitsOfInterrogationByState( + interrogationId, + surveyUnitModelsForInterrogationId, + surveyUnitModelsByInterrogationIdAndState + ); + } + + return surveyUnitModelsByInterrogationIdAndState; + } + + private static void addSurveyUnitsOfInterrogationByState( + String interrogationId, + List surveyUnitModelsForInterrogationId, + Map> surveyUnitModelsByInterrogationIdAndState + ) { + surveyUnitModelsByInterrogationIdAndState.put(interrogationId, new HashMap<>()); + for(SurveyUnitModel surveyUnitOfInterrogation : surveyUnitModelsForInterrogationId){ + if(surveyUnitOfInterrogation.getState() == null){ + continue; + } + surveyUnitModelsByInterrogationIdAndState.get(interrogationId).put( + surveyUnitOfInterrogation.getState(), + surveyUnitOfInterrogation + ); + } } } diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java index 72904b0df..867810c46 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java @@ -12,14 +12,17 @@ import fr.insee.genesis.domain.utils.GroupUtils; import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.modelefiliere.RawResponseDto; +import jakarta.validation.constraints.NotNull; import lombok.extern.slf4j.Slf4j; import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import static fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService.getValueString; @@ -67,7 +70,7 @@ public List convertRawResponseAndCollectEmptyModels( ) { List surveyUnitModels = new ArrayList<>(); - Map lastSurveyUnitModelsByInterrogationId = getLastSurveyUnitModels( + Map> lastSurveyUnitModelsByInterrogationIdAndState = getLastSurveyUnitModels( collectionInstrumentId, rawResponseModels.stream().map(RawResponseModel::interrogationId).collect(Collectors.toList()) ); @@ -75,17 +78,28 @@ public List convertRawResponseAndCollectEmptyModels( for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) { for (RawResponseModel rawResponseModel : rawResponseModels) { SurveyUnitModel surveyUnitModel = buildSurveyUnitModel(rawResponseModel, dataState); + SurveyUnitModel lastSurveyUnitModelForDataState = null; + if(lastSurveyUnitModelsByInterrogationIdAndState.containsKey(rawResponseModel.interrogationId())){ + lastSurveyUnitModelForDataState = lastSurveyUnitModelsByInterrogationIdAndState + .get(rawResponseModel.interrogationId()) + .get(dataState); + } convertCollectedVariables( rawResponseModel, - lastSurveyUnitModelsByInterrogationId.get(rawResponseModel.interrogationId()), + lastSurveyUnitModelForDataState, surveyUnitModel, dataState, variablesMap ); if (dataState == DataState.COLLECTED) { - convertExternalVariables(rawResponseModel, surveyUnitModel, variablesMap); + convertExternalVariables( + rawResponseModel, + lastSurveyUnitModelForDataState, + surveyUnitModel, + variablesMap + ); } boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() @@ -148,9 +162,13 @@ private void convertCollectedVariables( ) { Map dataMap = rawResponseModel.payload(); dataMap = JsonUtils.asMap(dataMap.get("data")); - Map collectedMap = JsonUtils.asMap(dataMap.get("COLLECTED")); + Map collectedMap = new HashMap<>(JsonUtils.asMap(dataMap.get("COLLECTED"))); + + if(lastSurveyUnitModel != null && lastSurveyUnitModel.getState().equals(dataState)) { + fillRawDataMapWithAbsentVariables(lastSurveyUnitModel, collectedMap, dataState); + } - if (collectedMap == null || collectedMap.isEmpty()) { + if (collectedMap.isEmpty()) { if (dataState == DataState.COLLECTED) { log.warn("No collected data for interrogation {}", rawResponseModel.interrogationId()); } @@ -188,62 +206,98 @@ public static void processCollectedVariableForState( Map variableStates = JsonUtils.asMap(entry.getValue()); //If variable value absent for state or null value if (variableStates == null || variableStates.get(stateKey) == null) { - convertNullVar(entry, lastSurveyUnitModel, variableModelList); + convertNullVar(entry.getKey(), lastSurveyUnitModel, 1, variablesMap, variableModelList, true); return; } Object value = variableStates.get(stateKey); if (value instanceof List list) { - convertListVar(list, entry, variablesMap, variableModelList); + convertListVar(list, lastSurveyUnitModel, entry, variablesMap, variableModelList, true); return; } - convertOneVar(entry, getValueString(value), variablesMap, 1, variableModelList); + convertOneVar(entry.getKey(), getValueString(value), variablesMap, 1, variableModelList); } private static void convertNullVar( - Map.Entry variableEntry, + String variableName, @Nullable SurveyUnitModel lastSurveyUnitModel, - List destination + int iteration, + VariablesMap variablesMap, + List destination, + boolean isCollected //true if in collected variables, false if in external ) { - if(lastSurveyUnitModel == null){ + //Do nothing if last response is null or variable is not present in last response + if (lastSurveyUnitModel == null || isVariableAbsentInSurveyUnitModel( + variableName, iteration, lastSurveyUnitModel, isCollected + )) { return; } - //TODO + convertOneVar(variableName, null, variablesMap, iteration, destination); + } + private static boolean isVariableAbsentInSurveyUnitModel( + String variableName, + int iteration, + SurveyUnitModel lastSurveyUnitModel, + boolean isCollected + ) { + //Check in collected variables + if(isCollected){ + return lastSurveyUnitModel.getCollectedVariables().stream().noneMatch( + variableModel -> variableModel.varId().equals(variableName) + && variableModel.iteration().equals(iteration) + ); + } + //Check in external variables + return lastSurveyUnitModel.getExternalVariables().stream().noneMatch( + variableModel -> variableModel.varId().equals(variableName) + && variableModel.iteration().equals(iteration) + ); } private static void convertListVar( Object valuesForState, + @Nullable SurveyUnitModel lastSurveyUnit, Map.Entry variableEntry, VariablesMap variablesMap, - List destination + List destination, + boolean isCollected ) { List values = JsonUtils.asStringList(valuesForState); if (!values.isEmpty()) { int iteration = 1; for (String value : values) { - if (value != null && !value.isEmpty()) { - convertOneVar(variableEntry, value, variablesMap, iteration, destination); + if(value == null || value.isEmpty()){ + convertNullVar( + variableEntry.getKey(), + lastSurveyUnit, + iteration, + variablesMap, + destination, + isCollected); + iteration++; + continue; } + convertOneVar(variableEntry.getKey(), value, variablesMap, iteration, destination); iteration++; } } } private static void convertOneVar( - Map.Entry variableEntry, + String variableName, String value, VariablesMap variablesMap, int iteration, List destination ) { VariableModel variableModel = VariableModel.builder() - .varId(variableEntry.getKey()) + .varId(variableName) .value(value) - .scope(getIdLoop(variablesMap, variableEntry.getKey())) + .scope(getIdLoop(variablesMap, variableName)) .iteration(iteration) - .parentId(GroupUtils.getParentGroupName(variableEntry.getKey(), variablesMap)) + .parentId(GroupUtils.getParentGroupName(variableName, variablesMap)) .build(); destination.add(variableModel); @@ -260,36 +314,53 @@ private static String getIdLoop(VariablesMap variablesMap, String variableName) private void convertExternalVariables( RawResponseModel rawResponseModel, + @Nullable SurveyUnitModel lastSurveyUnitModel, SurveyUnitModel dstSurveyUnitModel, VariablesMap variablesMap ) { Map dataMap = rawResponseModel.payload(); dataMap = JsonUtils.asMap(dataMap.get("data")); - Map externalMap = JsonUtils.asMap(dataMap.get("EXTERNAL")); + Map externalMap = null; + if(dataMap.containsKey("EXTERNAL")) { + externalMap = new HashMap<>(JsonUtils.asMap(dataMap.get("EXTERNAL"))); + } + + if(lastSurveyUnitModel != null){ + fillRawDataMapWithAbsentVariables(lastSurveyUnitModel, externalMap, null); + } - if (externalMap != null && !externalMap.isEmpty()) { + if (externalMap != null) { for (Map.Entry externalVariableEntry : externalMap.entrySet()) { Object valueObject = externalVariableEntry.getValue(); + if(valueObject == null){ + convertNullVar( + externalVariableEntry.getKey(), + lastSurveyUnitModel, + 1, + variablesMap, + dstSurveyUnitModel.getExternalVariables(), + false); + continue; + } if (valueObject instanceof List) { convertListVar( valueObject, + lastSurveyUnitModel, externalVariableEntry, variablesMap, - dstSurveyUnitModel.getExternalVariables() + dstSurveyUnitModel.getExternalVariables(), + false ); continue; } - - if (valueObject != null) { - convertOneVar( - externalVariableEntry, - valueObject.toString(), - variablesMap, - 1, - dstSurveyUnitModel.getExternalVariables() - ); - } + convertOneVar( + externalVariableEntry.getKey(), + valueObject.toString(), + variablesMap, + 1, + dstSurveyUnitModel.getExternalVariables() + ); } } } @@ -354,4 +425,172 @@ private static Object getValueForState( private static boolean isInvalidPairwiseVariable(Object value, VariablesMap variablesMap) { return !(value instanceof List) || !variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1); } + + /** + *

Adds missing variables/iterations to raw data map based on last survey unit in database

+ *

e.g. raw: var1 absent AND response: var1 present -> var1 added with null value

+ * + * @param rawDataMap dataMap (COLLECTED or EXTERNAL) + * @param dataState null if EXTERNAL + */ + private static void fillRawDataMapWithAbsentVariables( + @NotNull SurveyUnitModel lastSurveyUnit, + Map rawDataMap, + @Nullable DataState dataState + ) { + List lastSurveyUnitVariables = dataState == null ? + lastSurveyUnit.getExternalVariables() : lastSurveyUnit.getCollectedVariables(); + + //Extract variable names + Set lastSurveyUnitVariablesNames = lastSurveyUnitVariables.stream() + .map(VariableModel::varId).collect(Collectors.toSet()); + + for(String lastSurveyUnitVariableName : lastSurveyUnitVariablesNames){ + List variableModelsForSameVariable = lastSurveyUnitVariables.stream().filter( + variableAlreadyPresent -> + variableAlreadyPresent.varId().equals(lastSurveyUnitVariableName) + ).toList(); + + //If multiple iterations + if(variableModelsForSameVariable.size() > 1){ + fillRawDataMapWithIterations( + variableModelsForSameVariable, + rawDataMap, + dataState + ); + continue; + } + addNullVariableToRawDataMapIfAbsent(lastSurveyUnitVariableName, rawDataMap); + } + } + + /** + * Checks each iteration of a last response variable and adds null to raw data if iteration is absent + * @param alreadyPresentVariableIterations Variables models for same variable in last response + * @param rawDataMap data map (COLLECTED or EXTERNAL) + * @param dataState null if EXTERNAL + */ + @SuppressWarnings("unchecked") + private static void fillRawDataMapWithIterations( + List alreadyPresentVariableIterations, + Map rawDataMap, + @Nullable DataState dataState + ){ + if(alreadyPresentVariableIterations == null || alreadyPresentVariableIterations.isEmpty()){ + return; + } + String variableName = alreadyPresentVariableIterations.getFirst().varId(); + + int maxResponseIteration = alreadyPresentVariableIterations.stream() + .map(VariableModel::iteration) + .max(Integer::compareTo).orElse(0); + + List rawValuesList = getRawDataListForVariable( + variableName, + rawDataMap, + dataState + ); + + //Add null element at end of raw list until max iteration + fillListUntilMaxIteration(rawValuesList, maxResponseIteration); + + //Replace list in raw data map + if(dataState == null){ + rawDataMap.put(variableName, rawValuesList); + return; + } + Map stateMap = (Map) rawDataMap.get(variableName); + stateMap.put(dataState.toString(), rawValuesList); + } + + /** + * @param variableName Name of the variable contained in last response + * @param rawDataMap data map (COLLECTED or EXTERNAL) + * @param dataState null if EXTERNAL + * @return The list of values to put in raw data + */ + @SuppressWarnings("unchecked") + private static List getRawDataListForVariable( + String variableName, + Map rawDataMap, + DataState dataState + ) { + //Instantiate list in rawDataMap if variable exists in response but not in raw + if(!rawDataMap.containsKey(variableName)){ + return instantiateEmptyListInRawDataMap(variableName, rawDataMap, dataState); + } + + if(dataState == null){ + return rawDataMap.get(variableName) instanceof List ? + (List) rawDataMap.get(variableName) + //If single value in raw but multiple in response, return a list with the single raw value + : instantiateListInRawDataMapWithSingleValue(variableName, rawDataMap, null); + } + Map stateMap = (Map) rawDataMap.get(variableName); + return stateMap.get(dataState.toString()) instanceof List ? + (List) stateMap.get(dataState.toString()) + : instantiateListInRawDataMapWithSingleValue(variableName, rawDataMap, dataState); + } + + /** + * @param variableName Name of the variable to add in map + * @param rawDataMap COLLECTED or EXTERNAL map to fill + * @param dataState null if in EXTERNAL map, COLLECTED if not + * @return the instantiated list + */ + private static List instantiateEmptyListInRawDataMap( + String variableName, + Map rawDataMap, + @Nullable DataState dataState + ) { + List valueList = new ArrayList<>(); + if(dataState == null){ + rawDataMap.put(variableName, valueList); + return valueList; + } + Map statesMap = new HashMap<>(); + statesMap.put(dataState.toString(), valueList); + rawDataMap.put(variableName, statesMap); + + return valueList; + } + + @SuppressWarnings("unchecked") + private static List instantiateListInRawDataMapWithSingleValue( + String variableName, + Map rawDataMap, + DataState dataState + ) { + List singleValueList = new ArrayList<>(1); + if(dataState == null){ + singleValueList.add(rawDataMap.get(variableName)); + return singleValueList; + } + Map stateMap = (Map) rawDataMap.get(variableName); + singleValueList.add(stateMap.get(dataState.toString())); + return singleValueList; + } + + /** + * Fills the list of values with null until max iteration + * @param rawValuesList raw data values for a variable + * @param maxResponseIteration max iteration in last survey unit for said variable + */ + private static void fillListUntilMaxIteration(List rawValuesList, int maxResponseIteration) { + for(int i = 1; i <= maxResponseIteration; i++){ + if(rawValuesList.size() >= i){ + continue; + } + rawValuesList.addLast(null); + } + } + + private static void addNullVariableToRawDataMapIfAbsent( + String variableName, + Map rawDataMap + ) { + if (!rawDataMap.containsKey(variableName)){ + rawDataMap.put(variableName, null); + } + } } diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java index 6dba133dc..49a664e5e 100644 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java @@ -1,9 +1,9 @@ package fr.insee.genesis.domain.converter.rawdata; +import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -14,11 +14,11 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; -import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.eq; @ExtendWith(MockitoExtension.class) @@ -28,9 +28,9 @@ class RawDataConverterTest { private SurveyUnitService surveyUnitService; @InjectMocks - private final RawDataConverter rawDataConverterTestImpl = new RawDataConverter() { + private final RawDataConverter rawDataConverterTestImpl = new RawDataConverter(surveyUnitService) { @Override - protected Map getLastSurveyUnitModels(String questionnaireOrCollectionInstrumentId, List interrogationIds) { + protected Map> getLastSurveyUnitModels(String questionnaireOrCollectionInstrumentId, List interrogationIds) { return super.getLastSurveyUnitModels(questionnaireOrCollectionInstrumentId, interrogationIds); } }; @@ -40,31 +40,31 @@ protected Map getLastSurveyUnitModels(String questionna @DisplayName("getLastSurveyUnitModels tests") class getLastSurveyUnitModelsTests{ @Test - @DisplayName("Should return map with interrogation ids as key") + @DisplayName("Should return map of map with interrogation ids and states as keys") void getLastSurveyUnitModelsTest() { //GIVEN String questionnaireId = "questionnaire"; - List surveyUnitModels = - List.of( - SurveyUnitModel.builder() - .collectionInstrumentId(questionnaireId) - .interrogationId("Interrogation1") - .build(), - SurveyUnitModel.builder() - .collectionInstrumentId(questionnaireId) - .interrogationId("Interrogation2") - .build() - ); + Set interrogationIds = Set.of("Interrogation1", "Interrogation2"); + List surveyUnitModels = new ArrayList<>(); + for(String interrogationId : interrogationIds){ + surveyUnitModels.add( + SurveyUnitModel.builder() + .collectionInstrumentId(questionnaireId) + .state(DataState.COLLECTED) + .interrogationId(interrogationId) + .build() + ); + } - Mockito.when(surveyUnitService.findLatestByInterrogationIds( - eq(questionnaireId), anySet()) + Mockito.when(surveyUnitService.findLatestByInterrogationIds(questionnaireId, interrogationIds) ).thenReturn(surveyUnitModels); //WHEN - Map resultMap = rawDataConverterTestImpl.getLastSurveyUnitModels( + + Map> resultMap = rawDataConverterTestImpl.getLastSurveyUnitModels( questionnaireId, - List.of("Interrogation1","Interrogation2") + interrogationIds.stream().toList() ); @@ -81,9 +81,14 @@ void getLastSurveyUnitModelsTest() { ); //Check resulted map key and values content - Assertions.assertThat(resultMap).containsOnlyKeys("Interrogation1","Interrogation2"); - Assertions.assertThat(resultMap.get("Interrogation1").getInterrogationId()).isEqualTo("Interrogation1"); - Assertions.assertThat(resultMap.get("Interrogation2").getInterrogationId()).isEqualTo("Interrogation2"); + Assertions.assertThat(resultMap).containsOnlyKeys(interrogationIds); + for(String interrogationId : interrogationIds){ + Map surveyUnitModelsOfInterrogationId = resultMap.get(interrogationId); + Assertions.assertThat(surveyUnitModelsOfInterrogationId).containsOnlyKeys(DataState.COLLECTED); + Assertions.assertThat( + surveyUnitModelsOfInterrogationId.get(DataState.COLLECTED).getInterrogationId() + ).isEqualTo(interrogationId); + } } } } \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java index 7dcc15ab3..5e5f1424e 100644 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java @@ -38,6 +38,7 @@ class RawResponseRawDataConverterTest { private static final String COLLECTION_INSTRUMENT_ID = "collection-instrument-id"; + private static final String INTERROGATION_ID = "testInterrogation"; private static final Mode MODE = Mode.WEB; @@ -298,7 +299,6 @@ void shouldConvertPairwiseCollectedVariables() { class NullVariablesTests { //NullVariablesTests constants - private static final String INTERROGATION_ID = "testInterrogation"; private static final String COLLECTED_VARIABLE_NAME = "VAR1"; private static final String COLLECTED_VARIABLE_VALUE = "test"; private static final String EXTERNAL_VARIABLE_NAME = "EXTVAR1"; @@ -433,16 +433,16 @@ void shouldConvertNullValueIfNonNullOneValue(){ )); //Raw response with null - Map collectedVariablesMap = new LinkedHashMap<>(); + Map collectedVariableStatesMap = new LinkedHashMap<>(); Map externalVariablesMap = new LinkedHashMap<>(); - collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, null); + collectedVariableStatesMap.put("COLLECTED", null); externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, null); RawResponseModel rawResponseModel = rawResponseModel( payloadWith( - Map.of("COLLECTED", collectedVariablesMap), + Map.of(COLLECTED_VARIABLE_NAME, collectedVariableStatesMap), externalVariablesMap ) ); @@ -468,24 +468,22 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ //Already existing survey unit with non-null variables SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); - surveyUnitModel.getCollectedVariables().add( - VariableModel.builder() - .varId(COLLECTED_VARIABLE_NAME) - .value("VALUE2") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(2) - .build() - ); - surveyUnitModel.getExternalVariables().add( - VariableModel.builder() - .varId(EXTERNAL_VARIABLE_NAME) - .value("ext2") - .state(DataState.COLLECTED) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(2) - .build() - ); + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value("VALUE2") + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value("ext2") + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) .thenReturn(List.of( @@ -493,13 +491,13 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ )); //Raw response with null second iteration - Map collectedVariablesMap = new LinkedHashMap<>(); + Map collectedVariableStatesMap = new LinkedHashMap<>(); Map externalVariablesMap = new LinkedHashMap<>(); List variablesStrings = new ArrayList<>(); variablesStrings.add(COLLECTED_VARIABLE_VALUE); variablesStrings.add(null); - collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", variablesStrings)); + collectedVariableStatesMap.put("COLLECTED", variablesStrings); variablesStrings = new ArrayList<>(); variablesStrings.add(EXTERNAL_VARIABLE_VALUE); @@ -509,7 +507,7 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ RawResponseModel rawResponseModel = rawResponseModel( payloadWith( - Map.of("COLLECTED", collectedVariablesMap), + Map.of(COLLECTED_VARIABLE_NAME, collectedVariableStatesMap), externalVariablesMap ) ); @@ -529,7 +527,7 @@ void shouldConvertNullValueIfNonNullMultipleValues(){ @ParameterizedTest @ValueSource(booleans = {false, true}) - @DisplayName("Should keep null value if variable null or absent") + @DisplayName("Should keep null value if variable null or absent in raw") void shouldKeepNull(boolean isNewVariablesPresent){ //GIVEN VariablesMap variablesMap = mock(VariablesMap.class); @@ -538,6 +536,7 @@ void shouldKeepNull(boolean isNewVariablesPresent){ SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) .interrogationId(INTERROGATION_ID) + .state(DataState.COLLECTED) .collectedVariables(new ArrayList<>()) .externalVariables(new ArrayList<>()) .build(); @@ -545,7 +544,6 @@ void shouldKeepNull(boolean isNewVariablesPresent){ VariableModel.builder() .varId(COLLECTED_VARIABLE_NAME) .value(null) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(1) .build() @@ -554,7 +552,6 @@ void shouldKeepNull(boolean isNewVariablesPresent){ VariableModel.builder() .varId(EXTERNAL_VARIABLE_NAME) .value(null) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(1) .build() @@ -566,16 +563,16 @@ void shouldKeepNull(boolean isNewVariablesPresent){ )); //Raw response with null variables or no variable at all - Map collectedVariablesMap = new LinkedHashMap<>(); + Map collectedVariableStatesMap = new LinkedHashMap<>(); Map externalVariablesMap = new LinkedHashMap<>(); if(isNewVariablesPresent) { - collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, null); + collectedVariableStatesMap.put("COLLECTED", null); externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, null); } RawResponseModel rawResponseModel = rawResponseModel( payloadWith( - Map.of("COLLECTED", collectedVariablesMap), + Map.of(COLLECTED_VARIABLE_NAME, collectedVariableStatesMap), externalVariablesMap ) ); @@ -606,7 +603,6 @@ void shouldKeepNullIteration(boolean isSecondIterationPresent){ VariableModel.builder() .varId(COLLECTED_VARIABLE_NAME) .value(null) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(2) .build() @@ -615,7 +611,6 @@ void shouldKeepNullIteration(boolean isSecondIterationPresent){ VariableModel.builder() .varId(EXTERNAL_VARIABLE_NAME) .value(null) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(2) .build() @@ -627,7 +622,7 @@ void shouldKeepNullIteration(boolean isSecondIterationPresent){ )); //Raw response with null second iteration or no second iteration at all - Map collectedVariablesMap = new LinkedHashMap<>(); + Map collectedVariableStatesMap = new LinkedHashMap<>(); Map externalVariablesMap = new LinkedHashMap<>(); List collectedVariablesValues = new ArrayList<>(); @@ -641,12 +636,12 @@ void shouldKeepNullIteration(boolean isSecondIterationPresent){ externalVariablesValues.add(null); } - collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", collectedVariablesValues)); + collectedVariableStatesMap.put("COLLECTED", collectedVariablesValues); externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, externalVariablesValues); RawResponseModel rawResponseModel = rawResponseModel( payloadWith( - Map.of("COLLECTED", collectedVariablesMap), + Map.of(COLLECTED_VARIABLE_NAME, collectedVariableStatesMap), externalVariablesMap ) ); @@ -669,6 +664,7 @@ private SurveyUnitModel getSurveyUnitModel() { SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) .interrogationId(INTERROGATION_ID) + .state(DataState.COLLECTED) .collectedVariables(new ArrayList<>()) .externalVariables(new ArrayList<>()) .build(); @@ -677,7 +673,6 @@ private SurveyUnitModel getSurveyUnitModel() { VariableModel.builder() .varId(COLLECTED_VARIABLE_NAME) .value(COLLECTED_VARIABLE_VALUE) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(1) .build() @@ -686,7 +681,6 @@ private SurveyUnitModel getSurveyUnitModel() { VariableModel.builder() .varId(EXTERNAL_VARIABLE_NAME) .value(EXTERNAL_VARIABLE_VALUE) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(1) .build() @@ -753,7 +747,7 @@ private RawResponseModel rawResponseModel(Map payload) { RawResponseModel rawResponseModel = RawResponseModel.builder() .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) .mode(MODE) - .interrogationId("interrogation-id") + .interrogationId(INTERROGATION_ID) .recordDate(LocalDateTime.parse("2025-01-01T10:00:00")) .payload(payload) .build(); From d550698cf057e8ac7f373b3baaf6d43c8f79b703 Mon Sep 17 00:00:00 2001 From: Alexis Szmundy Date: Fri, 5 Jun 2026 17:18:00 +0200 Subject: [PATCH 07/12] refactor: moved absent variable management to abstract --- .../converter/rawdata/RawDataConverter.java | 173 ++++++++++++++++++ .../rawdata/RawResponseRawDataConverter.java | 170 ----------------- 2 files changed, 173 insertions(+), 170 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java index 170759721..da30e22d1 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java @@ -2,15 +2,20 @@ import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import jakarta.validation.constraints.NotNull; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; @Component public abstract class RawDataConverter { @@ -72,4 +77,172 @@ private static void addSurveyUnitsOfInterrogationByState( ); } } + + /** + *

Adds missing variables/iterations to raw data map based on last survey unit in database

+ *

e.g. raw: var1 absent AND response: var1 present -> var1 added with null value

+ * + * @param rawDataMap dataMap (COLLECTED or EXTERNAL) + * @param dataState null if EXTERNAL + */ + protected static void fillRawDataMapWithAbsentVariables( + @NotNull SurveyUnitModel lastSurveyUnit, + Map rawDataMap, + @Nullable DataState dataState + ) { + List lastSurveyUnitVariables = dataState == null ? + lastSurveyUnit.getExternalVariables() : lastSurveyUnit.getCollectedVariables(); + + //Extract variable names + Set lastSurveyUnitVariablesNames = lastSurveyUnitVariables.stream() + .map(VariableModel::varId).collect(Collectors.toSet()); + + for(String lastSurveyUnitVariableName : lastSurveyUnitVariablesNames){ + List variableModelsForSameVariable = lastSurveyUnitVariables.stream().filter( + variableAlreadyPresent -> + variableAlreadyPresent.varId().equals(lastSurveyUnitVariableName) + ).toList(); + + //If multiple iterations + if(variableModelsForSameVariable.size() > 1){ + fillRawDataMapWithIterations( + variableModelsForSameVariable, + rawDataMap, + dataState + ); + continue; + } + addNullVariableToRawDataMapIfAbsent(lastSurveyUnitVariableName, rawDataMap); + } + } + + /** + * Checks each iteration of a last response variable and adds null to raw data if iteration is absent + * @param alreadyPresentVariableIterations Variables models for same variable in last response + * @param rawDataMap data map (COLLECTED or EXTERNAL) + * @param dataState null if EXTERNAL + */ + @SuppressWarnings("unchecked") + private static void fillRawDataMapWithIterations( + List alreadyPresentVariableIterations, + Map rawDataMap, + @Nullable DataState dataState + ){ + if(alreadyPresentVariableIterations == null || alreadyPresentVariableIterations.isEmpty()){ + return; + } + String variableName = alreadyPresentVariableIterations.getFirst().varId(); + + int maxResponseIteration = alreadyPresentVariableIterations.stream() + .map(VariableModel::iteration) + .max(Integer::compareTo).orElse(0); + + List rawValuesList = getRawDataListForVariable( + variableName, + rawDataMap, + dataState + ); + + //Add null element at end of raw list until max iteration + fillListUntilMaxIteration(rawValuesList, maxResponseIteration); + + //Replace list in raw data map + if(dataState == null){ + rawDataMap.put(variableName, rawValuesList); + return; + } + Map stateMap = (Map) rawDataMap.get(variableName); + stateMap.put(dataState.toString(), rawValuesList); + } + + /** + * @param variableName Name of the variable contained in last response + * @param rawDataMap data map (COLLECTED or EXTERNAL) + * @param dataState null if EXTERNAL + * @return The list of values to put in raw data + */ + @SuppressWarnings("unchecked") + private static List getRawDataListForVariable( + String variableName, + Map rawDataMap, + DataState dataState + ) { + //Instantiate list in rawDataMap if variable exists in response but not in raw + if(!rawDataMap.containsKey(variableName)){ + return instantiateEmptyListInRawDataMap(variableName, rawDataMap, dataState); + } + + if(dataState == null){ + return rawDataMap.get(variableName) instanceof List ? + (List) rawDataMap.get(variableName) + //If single value in raw but multiple in response, return a list with the single raw value + : instantiateListInRawDataMapWithSingleValue(variableName, rawDataMap, null); + } + Map stateMap = (Map) rawDataMap.get(variableName); + return stateMap.get(dataState.toString()) instanceof List ? + (List) stateMap.get(dataState.toString()) + : instantiateListInRawDataMapWithSingleValue(variableName, rawDataMap, dataState); + } + + /** + * @param variableName Name of the variable to add in map + * @param rawDataMap COLLECTED or EXTERNAL map to fill + * @param dataState null if in EXTERNAL map, COLLECTED if not + * @return the instantiated list + */ + private static List instantiateEmptyListInRawDataMap( + String variableName, + Map rawDataMap, + @Nullable DataState dataState + ) { + List valueList = new ArrayList<>(); + if(dataState == null){ + rawDataMap.put(variableName, valueList); + return valueList; + } + Map statesMap = new HashMap<>(); + statesMap.put(dataState.toString(), valueList); + rawDataMap.put(variableName, statesMap); + + return valueList; + } + + @SuppressWarnings("unchecked") + private static List instantiateListInRawDataMapWithSingleValue( + String variableName, + Map rawDataMap, + DataState dataState + ) { + List singleValueList = new ArrayList<>(1); + if(dataState == null){ + singleValueList.add(rawDataMap.get(variableName)); + return singleValueList; + } + Map stateMap = (Map) rawDataMap.get(variableName); + singleValueList.add(stateMap.get(dataState.toString())); + return singleValueList; + } + + /** + * Fills the list of values with null until max iteration + * @param rawValuesList raw data values for a variable + * @param maxResponseIteration max iteration in last survey unit for said variable + */ + private static void fillListUntilMaxIteration(List rawValuesList, int maxResponseIteration) { + for(int i = 1; i <= maxResponseIteration; i++){ + if(rawValuesList.size() >= i){ + continue; + } + rawValuesList.addLast(null); + } + } + + private static void addNullVariableToRawDataMapIfAbsent( + String variableName, + Map rawDataMap + ) { + if (!rawDataMap.containsKey(variableName)){ + rawDataMap.put(variableName, null); + } + } } diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java index 867810c46..cb66b3188 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java @@ -12,7 +12,6 @@ import fr.insee.genesis.domain.utils.GroupUtils; import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.modelefiliere.RawResponseDto; -import jakarta.validation.constraints.NotNull; import lombok.extern.slf4j.Slf4j; import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; @@ -22,7 +21,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import static fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService.getValueString; @@ -425,172 +423,4 @@ private static Object getValueForState( private static boolean isInvalidPairwiseVariable(Object value, VariablesMap variablesMap) { return !(value instanceof List) || !variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1); } - - /** - *

Adds missing variables/iterations to raw data map based on last survey unit in database

- *

e.g. raw: var1 absent AND response: var1 present -> var1 added with null value

- * - * @param rawDataMap dataMap (COLLECTED or EXTERNAL) - * @param dataState null if EXTERNAL - */ - private static void fillRawDataMapWithAbsentVariables( - @NotNull SurveyUnitModel lastSurveyUnit, - Map rawDataMap, - @Nullable DataState dataState - ) { - List lastSurveyUnitVariables = dataState == null ? - lastSurveyUnit.getExternalVariables() : lastSurveyUnit.getCollectedVariables(); - - //Extract variable names - Set lastSurveyUnitVariablesNames = lastSurveyUnitVariables.stream() - .map(VariableModel::varId).collect(Collectors.toSet()); - - for(String lastSurveyUnitVariableName : lastSurveyUnitVariablesNames){ - List variableModelsForSameVariable = lastSurveyUnitVariables.stream().filter( - variableAlreadyPresent -> - variableAlreadyPresent.varId().equals(lastSurveyUnitVariableName) - ).toList(); - - //If multiple iterations - if(variableModelsForSameVariable.size() > 1){ - fillRawDataMapWithIterations( - variableModelsForSameVariable, - rawDataMap, - dataState - ); - continue; - } - addNullVariableToRawDataMapIfAbsent(lastSurveyUnitVariableName, rawDataMap); - } - } - - /** - * Checks each iteration of a last response variable and adds null to raw data if iteration is absent - * @param alreadyPresentVariableIterations Variables models for same variable in last response - * @param rawDataMap data map (COLLECTED or EXTERNAL) - * @param dataState null if EXTERNAL - */ - @SuppressWarnings("unchecked") - private static void fillRawDataMapWithIterations( - List alreadyPresentVariableIterations, - Map rawDataMap, - @Nullable DataState dataState - ){ - if(alreadyPresentVariableIterations == null || alreadyPresentVariableIterations.isEmpty()){ - return; - } - String variableName = alreadyPresentVariableIterations.getFirst().varId(); - - int maxResponseIteration = alreadyPresentVariableIterations.stream() - .map(VariableModel::iteration) - .max(Integer::compareTo).orElse(0); - - List rawValuesList = getRawDataListForVariable( - variableName, - rawDataMap, - dataState - ); - - //Add null element at end of raw list until max iteration - fillListUntilMaxIteration(rawValuesList, maxResponseIteration); - - //Replace list in raw data map - if(dataState == null){ - rawDataMap.put(variableName, rawValuesList); - return; - } - Map stateMap = (Map) rawDataMap.get(variableName); - stateMap.put(dataState.toString(), rawValuesList); - } - - /** - * @param variableName Name of the variable contained in last response - * @param rawDataMap data map (COLLECTED or EXTERNAL) - * @param dataState null if EXTERNAL - * @return The list of values to put in raw data - */ - @SuppressWarnings("unchecked") - private static List getRawDataListForVariable( - String variableName, - Map rawDataMap, - DataState dataState - ) { - //Instantiate list in rawDataMap if variable exists in response but not in raw - if(!rawDataMap.containsKey(variableName)){ - return instantiateEmptyListInRawDataMap(variableName, rawDataMap, dataState); - } - - if(dataState == null){ - return rawDataMap.get(variableName) instanceof List ? - (List) rawDataMap.get(variableName) - //If single value in raw but multiple in response, return a list with the single raw value - : instantiateListInRawDataMapWithSingleValue(variableName, rawDataMap, null); - } - Map stateMap = (Map) rawDataMap.get(variableName); - return stateMap.get(dataState.toString()) instanceof List ? - (List) stateMap.get(dataState.toString()) - : instantiateListInRawDataMapWithSingleValue(variableName, rawDataMap, dataState); - } - - /** - * @param variableName Name of the variable to add in map - * @param rawDataMap COLLECTED or EXTERNAL map to fill - * @param dataState null if in EXTERNAL map, COLLECTED if not - * @return the instantiated list - */ - private static List instantiateEmptyListInRawDataMap( - String variableName, - Map rawDataMap, - @Nullable DataState dataState - ) { - List valueList = new ArrayList<>(); - if(dataState == null){ - rawDataMap.put(variableName, valueList); - return valueList; - } - Map statesMap = new HashMap<>(); - statesMap.put(dataState.toString(), valueList); - rawDataMap.put(variableName, statesMap); - - return valueList; - } - - @SuppressWarnings("unchecked") - private static List instantiateListInRawDataMapWithSingleValue( - String variableName, - Map rawDataMap, - DataState dataState - ) { - List singleValueList = new ArrayList<>(1); - if(dataState == null){ - singleValueList.add(rawDataMap.get(variableName)); - return singleValueList; - } - Map stateMap = (Map) rawDataMap.get(variableName); - singleValueList.add(stateMap.get(dataState.toString())); - return singleValueList; - } - - /** - * Fills the list of values with null until max iteration - * @param rawValuesList raw data values for a variable - * @param maxResponseIteration max iteration in last survey unit for said variable - */ - private static void fillListUntilMaxIteration(List rawValuesList, int maxResponseIteration) { - for(int i = 1; i <= maxResponseIteration; i++){ - if(rawValuesList.size() >= i){ - continue; - } - rawValuesList.addLast(null); - } - } - - private static void addNullVariableToRawDataMapIfAbsent( - String variableName, - Map rawDataMap - ) { - if (!rawDataMap.containsKey(variableName)){ - rawDataMap.put(variableName, null); - } - } } From 256b1e02c233d9082d6aa3ed18454bc3c58db2a8 Mon Sep 17 00:00:00 2001 From: Alexis Szmundy Date: Mon, 8 Jun 2026 15:11:21 +0200 Subject: [PATCH 08/12] refactor: moved common raw process logic into abstract class --- .../rawdata/LunaticJsonRawDataConverter.java | 160 +-------- .../converter/rawdata/RawDataConverter.java | 307 +++++++++++++++++- .../rawdata/RawResponseRawDataConverter.java | 290 +---------------- .../rawdata/LunaticJsonRawDataService.java | 11 - .../LunaticJsonRawDataConverterTest.java | 2 +- .../rawdata/RawDataConverterTest.java | 57 +++- .../LunaticJsonRawDataServiceTest.java | 47 --- 7 files changed, 382 insertions(+), 492 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java index 82201fc9f..eb46f789b 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java @@ -1,19 +1,13 @@ package fr.insee.genesis.domain.converter.rawdata; -import fr.insee.bpm.metadata.model.Variable; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.parser.rawdata.LunaticJsonRawDataPayloadParser; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; -import fr.insee.genesis.domain.utils.GroupUtils; -import fr.insee.genesis.domain.utils.JsonUtils; import lombok.extern.slf4j.Slf4j; -import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; import java.time.Instant; @@ -24,7 +18,7 @@ @Component @Slf4j -public class LunaticJsonRawDataConverter extends RawDataConverter{ +public class LunaticJsonRawDataConverter extends RawDataConverter { private final LunaticJsonRawDataPayloadParser payloadParser; @@ -70,7 +64,7 @@ public List convertRawDataAndCollectEmptyModels( .get(dataState); } - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + SurveyUnitModel newSurveyUnitModel = SurveyUnitModel.builder() .collectionInstrumentId(rawData.questionnaireId()) .mode(rawData.mode()) .interrogationId(rawData.interrogationId()) @@ -85,32 +79,39 @@ public List convertRawDataAndCollectEmptyModels( .externalVariables(new ArrayList<>()) .build(); - convertRawDataCollectedVariables( - rawData, + convertCollectedVariables( + rawData.data(), + rawData.interrogationId(), lastSurveyUnitModelForDataState, - surveyUnitModel, + newSurveyUnitModel, dataState, rawDataModelType, variablesMap ); if (dataState == DataState.COLLECTED) { - convertRawDataExternalVariables(rawData, surveyUnitModel, rawDataModelType, variablesMap); + convertExternalVariables( + rawData.data(), + lastSurveyUnitModelForDataState, + newSurveyUnitModel, + rawDataModelType, + variablesMap + ); } - boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() - && surveyUnitModel.getExternalVariables().isEmpty(); + boolean hasNoVariable = newSurveyUnitModel.getCollectedVariables().isEmpty() + && newSurveyUnitModel.getExternalVariables().isEmpty(); if (hasNoVariable) { - if (surveyUnitModel.getState() == DataState.COLLECTED) { + if (newSurveyUnitModel.getState() == DataState.COLLECTED) { log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawData.interrogationId()); } - emptySurveyUnitModels.add(surveyUnitModel); + emptySurveyUnitModels.add(newSurveyUnitModel); continue; } - surveyUnitModels.add(surveyUnitModel); + surveyUnitModels.add(newSurveyUnitModel); } } @@ -122,129 +123,4 @@ private static RawDataModelType getRawDataModelType(LunaticJsonRawDataModel rawD ? RawDataModelType.FILIERE : RawDataModelType.LEGACY; } - - private void convertRawDataCollectedVariables( - LunaticJsonRawDataModel srcRawData, - @Nullable SurveyUnitModel lastSurveyUnitModel, - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - RawDataModelType rawDataModelType, - VariablesMap variablesMap - ) { - Map dataMap = srcRawData.data(); - if (rawDataModelType == RawDataModelType.FILIERE) { - dataMap = JsonUtils.asMap(dataMap.get("data")); - } - - Map collectedMap = JsonUtils.asMap(dataMap.get("COLLECTED")); - - if (collectedMap == null || collectedMap.isEmpty()) { - if (dataState == DataState.COLLECTED) { - log.warn("No collected data for interrogation {}", srcRawData.interrogationId()); - } - return; - } - - String stateKey = dataState.toString(); - List destination = dstSurveyUnitModel.getCollectedVariables(); - - for (Map.Entry collectedVariable : collectedMap.entrySet()) { - RawResponseRawDataConverter.processCollectedVariableForState( - collectedVariable, - stateKey, - variablesMap, - lastSurveyUnitModel, - dstSurveyUnitModel, - destination - ); - } - } - - - private static void convertRawDataExternalVariables( - LunaticJsonRawDataModel srcRawData, - SurveyUnitModel dstSurveyUnitModel, - RawDataModelType rawDataModelType, - VariablesMap variablesMap - ) { - Map dataMap = srcRawData.data(); - if (rawDataModelType == RawDataModelType.FILIERE) { - dataMap = JsonUtils.asMap(dataMap.get("data")); - } - - Map externalMap = JsonUtils.asMap(dataMap.get("EXTERNAL")); - if (externalMap != null && !externalMap.isEmpty()) { - convertToExternalVar(dstSurveyUnitModel, variablesMap, externalMap); - } - } - - private static void convertToExternalVar( - SurveyUnitModel dstSurveyUnitModel, - VariablesMap variablesMap, - Map externalMap - ) { - for (Map.Entry externalVariableEntry : externalMap.entrySet()) { - Object valueObject = externalVariableEntry.getValue(); - - if (valueObject instanceof List) { - convertListVar(valueObject, externalVariableEntry, variablesMap, dstSurveyUnitModel.getExternalVariables()); - continue; - } - - if (valueObject != null) { - convertOneVar( - externalVariableEntry, - valueObject.toString(), - variablesMap, - 1, - dstSurveyUnitModel.getExternalVariables() - ); - } - } - } - - private static void convertListVar( - Object valuesForState, - Map.Entry collectedVariable, - VariablesMap variablesMap, - List destination - ) { - List values = JsonUtils.asStringList(valuesForState); - if (!values.isEmpty()) { - int iteration = 1; - for (String value : values) { - if (value != null && !value.isEmpty()) { - convertOneVar(collectedVariable, value, variablesMap, iteration, destination); - } - iteration++; - } - } - } - - private static void convertOneVar( - Map.Entry variableEntry, - String value, - VariablesMap variablesMap, - int iteration, - List destination - ) { - VariableModel variableModel = VariableModel.builder() - .varId(variableEntry.getKey()) - .value(value) - .scope(getIdLoop(variablesMap, variableEntry.getKey())) - .iteration(iteration) - .parentId(GroupUtils.getParentGroupName(variableEntry.getKey(), variablesMap)) - .build(); - - destination.add(variableModel); - } - - private static String getIdLoop(VariablesMap variablesMap, String variableName) { - Variable variable = variablesMap.getVariable(variableName); - if (variable == null) { - log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); - return Constants.ROOT_GROUP_NAME; - } - return variable.getGroupName(); - } } diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java index da30e22d1..aa9da84a7 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java @@ -1,14 +1,22 @@ package fr.insee.genesis.domain.converter.rawdata; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.domain.utils.GroupUtils; +import fr.insee.genesis.domain.utils.JsonUtils; import jakarta.validation.constraints.NotNull; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -17,9 +25,10 @@ import java.util.Set; import java.util.stream.Collectors; +@Slf4j @Component public abstract class RawDataConverter { - private SurveyUnitService surveyUnitService; + private final SurveyUnitService surveyUnitService; @Autowired public RawDataConverter(SurveyUnitService surveyUnitService) { @@ -85,7 +94,7 @@ private static void addSurveyUnitsOfInterrogationByState( * @param rawDataMap dataMap (COLLECTED or EXTERNAL) * @param dataState null if EXTERNAL */ - protected static void fillRawDataMapWithAbsentVariables( + private static void fillRawDataMapWithAbsentVariables( @NotNull SurveyUnitModel lastSurveyUnit, Map rawDataMap, @Nullable DataState dataState @@ -245,4 +254,298 @@ private static void addNullVariableToRawDataMapIfAbsent( rawDataMap.put(variableName, null); } } + + protected void convertCollectedVariables( + Map dataOrPayloadMap, + String interrogationId, + @Nullable SurveyUnitModel lastSurveyUnitModel, + SurveyUnitModel dstSurveyUnitModel, + DataState dataState, + RawDataModelType rawDataModelType, + VariablesMap variablesMap + ) { + //Get data map from payload + Map dataMap = dataOrPayloadMap; + if (rawDataModelType == RawDataModelType.FILIERE) { + dataMap = JsonUtils.asMap(dataOrPayloadMap.get("data")); + } + Map collectedMap = new HashMap<>(JsonUtils.asMap(dataMap.get("COLLECTED"))); + + if(lastSurveyUnitModel != null && lastSurveyUnitModel.getState().equals(dataState)) { + fillRawDataMapWithAbsentVariables(lastSurveyUnitModel, collectedMap, dataState); + } + + if (collectedMap.isEmpty()) { + if (dataState == DataState.COLLECTED) { + log.warn("No collected data for interrogation {}", interrogationId); + } + return; + } + + String stateKey = dataState.toString(); + List collectedVariables = dstSurveyUnitModel.getCollectedVariables(); + + for (Map.Entry entry : collectedMap.entrySet()) { + processCollectedVariableForState( + entry, + stateKey, + variablesMap, + lastSurveyUnitModel, + dstSurveyUnitModel, + collectedVariables + ); + } + } + + static void processCollectedVariableForState( + Map.Entry entry, + String stateKey, + VariablesMap variablesMap, + @Nullable SurveyUnitModel lastSurveyUnitModel, + SurveyUnitModel dstSurveyUnitModel, + List variableModelList + ) { + if (Constants.PAIRWISES.equals(entry.getKey())) { + handlePairwiseCollectedVariable(entry, DataState.valueOf(stateKey), variablesMap, dstSurveyUnitModel); + return; + } + + Map variableStates = JsonUtils.asMap(entry.getValue()); + //If variable value absent for state or null value + if (variableStates == null || variableStates.get(stateKey) == null) { + convertNullVar(entry.getKey(), lastSurveyUnitModel, 1, variablesMap, variableModelList, true); + return; + } + + Object value = variableStates.get(stateKey); + if (value instanceof List list) { + convertListVar(list, lastSurveyUnitModel, entry, variablesMap, variableModelList, true); + return; + } + convertOneVar(entry.getKey(), getValueString(value), variablesMap, 1, variableModelList); + } + + @SuppressWarnings("unchecked") + static void handlePairwiseCollectedVariable( + Map.Entry collectedVariable, + DataState dataState, + VariablesMap variablesMap, + SurveyUnitModel dstSurveyUnitModel + ) { + Object value = getValueForState(collectedVariable, dataState.toString()); + + if (isInvalidPairwiseVariable(value, variablesMap)) { + return; + } + + List individuals = (List) value; + String groupName = variablesMap.getVariable(Constants.PAIRWISE_PREFIX + 1).getGroupName(); + + for (int individualIndex = 0; individualIndex < individuals.size(); individualIndex++) { + List individualLinks = (List) individuals.get(individualIndex); + + for (int linkIndex = 1; linkIndex < Constants.MAX_LINKS_ALLOWED; linkIndex++) { + dstSurveyUnitModel.getCollectedVariables().add( + buildPairwiseVariable(individualLinks, linkIndex, individualIndex + 1, groupName) + ); + } + } + } + + private static VariableModel buildPairwiseVariable( + List individualLinks, + int linkIndex, + int iteration, + String groupName + ) { + String value = Constants.NO_PAIRWISE_VALUE; + + if (linkIndex <= individualLinks.size()) { + String v = individualLinks.get(linkIndex - 1); + value = (v == null || v.isBlank()) ? Constants.SAME_AXIS_VALUE : v; + } + + return VariableModel.builder() + .varId(Constants.PAIRWISE_PREFIX + linkIndex) + .value(value) + .scope(groupName) + .iteration(iteration) + .parentId(Constants.ROOT_GROUP_NAME) + .build(); + } + + private static Object getValueForState( + Map.Entry collectedVariable, + String stateKey + ) { + Map states = JsonUtils.asMap(collectedVariable.getValue()); + return states != null ? states.get(stateKey) : null; + } + + private static boolean isInvalidPairwiseVariable(Object value, VariablesMap variablesMap) { + return !(value instanceof List) || !variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1); + } + + private static void convertNullVar( + String variableName, + @Nullable SurveyUnitModel lastSurveyUnitModel, + int iteration, + VariablesMap variablesMap, + List destination, + boolean isCollected //true if in collected variables, false if in external + ) { + //Do nothing if last response is null or variable is not present in last response + if (lastSurveyUnitModel == null || isVariableAbsentInSurveyUnitModel( + variableName, iteration, lastSurveyUnitModel, isCollected + )) { + return; + } + convertOneVar(variableName, null, variablesMap, iteration, destination); + } + + private static boolean isVariableAbsentInSurveyUnitModel( + String variableName, + int iteration, + SurveyUnitModel lastSurveyUnitModel, + boolean isCollected + ) { + //Check in collected variables + if(isCollected){ + return lastSurveyUnitModel.getCollectedVariables().stream().noneMatch( + variableModel -> variableModel.varId().equals(variableName) + && variableModel.iteration().equals(iteration) + ); + } + //Check in external variables + return lastSurveyUnitModel.getExternalVariables().stream().noneMatch( + variableModel -> variableModel.varId().equals(variableName) + && variableModel.iteration().equals(iteration) + ); + } + + private static void convertOneVar( + String variableName, + String value, + VariablesMap variablesMap, + int iteration, + List destination + ) { + VariableModel variableModel = VariableModel.builder() + .varId(variableName) + .value(value) + .scope(getIdLoop(variablesMap, variableName)) + .iteration(iteration) + .parentId(GroupUtils.getParentGroupName(variableName, variablesMap)) + .build(); + + destination.add(variableModel); + } + + private static String getIdLoop(VariablesMap variablesMap, String variableName) { + Variable variable = variablesMap.getVariable(variableName); + if (variable == null) { + log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); + return Constants.ROOT_GROUP_NAME; + } + return variable.getGroupName(); + } + + private static void convertListVar( + Object valuesForState, + @Nullable SurveyUnitModel lastSurveyUnit, + Map.Entry variableEntry, + VariablesMap variablesMap, + List destination, + boolean isCollected + ) { + List values = JsonUtils.asStringList(valuesForState); + + if (!values.isEmpty()) { + int iteration = 1; + for (String value : values) { + if(value == null || value.isEmpty()){ + convertNullVar( + variableEntry.getKey(), + lastSurveyUnit, + iteration, + variablesMap, + destination, + isCollected); + iteration++; + continue; + } + convertOneVar(variableEntry.getKey(), value, variablesMap, iteration, destination); + iteration++; + } + } + } + + protected void convertExternalVariables( + Map dataOrPayloadMap, + @Nullable SurveyUnitModel lastSurveyUnitModel, + SurveyUnitModel dstSurveyUnitModel, + RawDataModelType rawDataModelType, + VariablesMap variablesMap + ) { + //Get data map from payload if filiere model + Map dataMap = dataOrPayloadMap; + if (rawDataModelType == RawDataModelType.FILIERE) { + dataMap = JsonUtils.asMap(dataOrPayloadMap.get("data")); + } + Map externalMap = null; + if(dataMap.containsKey("EXTERNAL")) { + externalMap = new HashMap<>(JsonUtils.asMap(dataMap.get("EXTERNAL"))); + } + + if(lastSurveyUnitModel != null){ + fillRawDataMapWithAbsentVariables(lastSurveyUnitModel, externalMap, null); + } + if(externalMap == null){ + return; + } + + for (Map.Entry externalVariableEntry : externalMap.entrySet()) { + Object valueObject = externalVariableEntry.getValue(); + + if (valueObject == null) { + convertNullVar( + externalVariableEntry.getKey(), + lastSurveyUnitModel, + 1, + variablesMap, + dstSurveyUnitModel.getExternalVariables(), + false); + continue; + } + if (valueObject instanceof List) { + convertListVar( + valueObject, + lastSurveyUnitModel, + externalVariableEntry, + variablesMap, + dstSurveyUnitModel.getExternalVariables(), + false + ); + continue; + } + convertOneVar( + externalVariableEntry.getKey(), + valueObject.toString(), + variablesMap, + 1, + dstSurveyUnitModel.getExternalVariables() + ); + } + } + + static String getValueString(Object value) { + if (value instanceof Double || value instanceof Float) { + BigDecimal bd = new BigDecimal(value.toString()); + return bd.stripTrailingZeros().toPlainString(); + } + if (value instanceof Number) { + return value.toString(); + } + return String.valueOf(value); + } } diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java index cb66b3188..b6c961dc4 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java @@ -1,30 +1,22 @@ package fr.insee.genesis.domain.converter.rawdata; -import fr.insee.bpm.metadata.model.Variable; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; -import fr.insee.genesis.domain.utils.GroupUtils; -import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.modelefiliere.RawResponseDto; import lombok.extern.slf4j.Slf4j; -import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; import java.time.Instant; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService.getValueString; - @Component @Slf4j public class RawResponseRawDataConverter extends RawDataConverter { @@ -84,18 +76,21 @@ public List convertRawResponseAndCollectEmptyModels( } convertCollectedVariables( - rawResponseModel, + rawResponseModel.payload(), + rawResponseModel.interrogationId(), lastSurveyUnitModelForDataState, surveyUnitModel, dataState, + RawDataModelType.FILIERE, variablesMap ); if (dataState == DataState.COLLECTED) { convertExternalVariables( - rawResponseModel, + rawResponseModel.payload(), lastSurveyUnitModelForDataState, surveyUnitModel, + RawDataModelType.FILIERE, variablesMap ); } @@ -150,277 +145,4 @@ private SurveyUnitModel buildSurveyUnitModel(RawResponseModel rawResponseModel, .externalVariables(new ArrayList<>()) .build(); } - - private void convertCollectedVariables( - RawResponseModel rawResponseModel, - @Nullable SurveyUnitModel lastSurveyUnitModel, - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - VariablesMap variablesMap - ) { - Map dataMap = rawResponseModel.payload(); - dataMap = JsonUtils.asMap(dataMap.get("data")); - Map collectedMap = new HashMap<>(JsonUtils.asMap(dataMap.get("COLLECTED"))); - - if(lastSurveyUnitModel != null && lastSurveyUnitModel.getState().equals(dataState)) { - fillRawDataMapWithAbsentVariables(lastSurveyUnitModel, collectedMap, dataState); - } - - if (collectedMap.isEmpty()) { - if (dataState == DataState.COLLECTED) { - log.warn("No collected data for interrogation {}", rawResponseModel.interrogationId()); - } - return; - } - - String stateKey = dataState.toString(); - List collectedVariables = dstSurveyUnitModel.getCollectedVariables(); - - for (Map.Entry entry : collectedMap.entrySet()) { - processCollectedVariableForState( - entry, - stateKey, - variablesMap, - lastSurveyUnitModel, - dstSurveyUnitModel, - collectedVariables - ); - } - } - - public static void processCollectedVariableForState( - Map.Entry entry, - String stateKey, - VariablesMap variablesMap, - @Nullable SurveyUnitModel lastSurveyUnitModel, - SurveyUnitModel dstSurveyUnitModel, - List variableModelList - ) { - if (Constants.PAIRWISES.equals(entry.getKey())) { - handlePairwiseCollectedVariable(entry, DataState.valueOf(stateKey), variablesMap, dstSurveyUnitModel); - return; - } - - Map variableStates = JsonUtils.asMap(entry.getValue()); - //If variable value absent for state or null value - if (variableStates == null || variableStates.get(stateKey) == null) { - convertNullVar(entry.getKey(), lastSurveyUnitModel, 1, variablesMap, variableModelList, true); - return; - } - - Object value = variableStates.get(stateKey); - if (value instanceof List list) { - convertListVar(list, lastSurveyUnitModel, entry, variablesMap, variableModelList, true); - return; - } - convertOneVar(entry.getKey(), getValueString(value), variablesMap, 1, variableModelList); - } - - private static void convertNullVar( - String variableName, - @Nullable SurveyUnitModel lastSurveyUnitModel, - int iteration, - VariablesMap variablesMap, - List destination, - boolean isCollected //true if in collected variables, false if in external - ) { - //Do nothing if last response is null or variable is not present in last response - if (lastSurveyUnitModel == null || isVariableAbsentInSurveyUnitModel( - variableName, iteration, lastSurveyUnitModel, isCollected - )) { - return; - } - convertOneVar(variableName, null, variablesMap, iteration, destination); - } - - private static boolean isVariableAbsentInSurveyUnitModel( - String variableName, - int iteration, - SurveyUnitModel lastSurveyUnitModel, - boolean isCollected - ) { - //Check in collected variables - if(isCollected){ - return lastSurveyUnitModel.getCollectedVariables().stream().noneMatch( - variableModel -> variableModel.varId().equals(variableName) - && variableModel.iteration().equals(iteration) - ); - } - //Check in external variables - return lastSurveyUnitModel.getExternalVariables().stream().noneMatch( - variableModel -> variableModel.varId().equals(variableName) - && variableModel.iteration().equals(iteration) - ); - } - - private static void convertListVar( - Object valuesForState, - @Nullable SurveyUnitModel lastSurveyUnit, - Map.Entry variableEntry, - VariablesMap variablesMap, - List destination, - boolean isCollected - ) { - List values = JsonUtils.asStringList(valuesForState); - - if (!values.isEmpty()) { - int iteration = 1; - for (String value : values) { - if(value == null || value.isEmpty()){ - convertNullVar( - variableEntry.getKey(), - lastSurveyUnit, - iteration, - variablesMap, - destination, - isCollected); - iteration++; - continue; - } - convertOneVar(variableEntry.getKey(), value, variablesMap, iteration, destination); - iteration++; - } - } - } - - private static void convertOneVar( - String variableName, - String value, - VariablesMap variablesMap, - int iteration, - List destination - ) { - VariableModel variableModel = VariableModel.builder() - .varId(variableName) - .value(value) - .scope(getIdLoop(variablesMap, variableName)) - .iteration(iteration) - .parentId(GroupUtils.getParentGroupName(variableName, variablesMap)) - .build(); - - destination.add(variableModel); - } - - private static String getIdLoop(VariablesMap variablesMap, String variableName) { - Variable variable = variablesMap.getVariable(variableName); - if (variable == null) { - log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); - return Constants.ROOT_GROUP_NAME; - } - return variable.getGroupName(); - } - - private void convertExternalVariables( - RawResponseModel rawResponseModel, - @Nullable SurveyUnitModel lastSurveyUnitModel, - SurveyUnitModel dstSurveyUnitModel, - VariablesMap variablesMap - ) { - Map dataMap = rawResponseModel.payload(); - dataMap = JsonUtils.asMap(dataMap.get("data")); - Map externalMap = null; - if(dataMap.containsKey("EXTERNAL")) { - externalMap = new HashMap<>(JsonUtils.asMap(dataMap.get("EXTERNAL"))); - } - - if(lastSurveyUnitModel != null){ - fillRawDataMapWithAbsentVariables(lastSurveyUnitModel, externalMap, null); - } - - if (externalMap != null) { - for (Map.Entry externalVariableEntry : externalMap.entrySet()) { - Object valueObject = externalVariableEntry.getValue(); - - if(valueObject == null){ - convertNullVar( - externalVariableEntry.getKey(), - lastSurveyUnitModel, - 1, - variablesMap, - dstSurveyUnitModel.getExternalVariables(), - false); - continue; - } - if (valueObject instanceof List) { - convertListVar( - valueObject, - lastSurveyUnitModel, - externalVariableEntry, - variablesMap, - dstSurveyUnitModel.getExternalVariables(), - false - ); - continue; - } - convertOneVar( - externalVariableEntry.getKey(), - valueObject.toString(), - variablesMap, - 1, - dstSurveyUnitModel.getExternalVariables() - ); - } - } - } - - @SuppressWarnings("unchecked") - static void handlePairwiseCollectedVariable( - Map.Entry collectedVariable, - DataState dataState, - VariablesMap variablesMap, - SurveyUnitModel dstSurveyUnitModel - ) { - Object value = getValueForState(collectedVariable, dataState.toString()); - - if (isInvalidPairwiseVariable(value, variablesMap)) { - return; - } - - List individuals = (List) value; - String groupName = variablesMap.getVariable(Constants.PAIRWISE_PREFIX + 1).getGroupName(); - - for (int individualIndex = 0; individualIndex < individuals.size(); individualIndex++) { - List individualLinks = (List) individuals.get(individualIndex); - - for (int linkIndex = 1; linkIndex < Constants.MAX_LINKS_ALLOWED; linkIndex++) { - dstSurveyUnitModel.getCollectedVariables().add( - buildPairwiseVariable(individualLinks, linkIndex, individualIndex + 1, groupName) - ); - } - } - } - - private static VariableModel buildPairwiseVariable( - List individualLinks, - int linkIndex, - int iteration, - String groupName - ) { - String value = Constants.NO_PAIRWISE_VALUE; - - if (linkIndex <= individualLinks.size()) { - String v = individualLinks.get(linkIndex - 1); - value = (v == null || v.isBlank()) ? Constants.SAME_AXIS_VALUE : v; - } - - return VariableModel.builder() - .varId(Constants.PAIRWISE_PREFIX + linkIndex) - .value(value) - .scope(groupName) - .iteration(iteration) - .parentId(Constants.ROOT_GROUP_NAME) - .build(); - } - - private static Object getValueForState( - Map.Entry collectedVariable, - String stateKey - ) { - Map states = JsonUtils.asMap(collectedVariable.getValue()); - return states != null ? states.get(stateKey) : null; - } - - private static boolean isInvalidPairwiseVariable(Object value, VariablesMap variablesMap) { - return !(value instanceof List) || !variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1); - } } diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 49c2416ca..ed6f4cbbe 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -375,15 +375,4 @@ public Page findRawDataByCampaignIdAndDate( ) { return lunaticJsonRawDataPersistencePort.findByCampaignIdAndDate(campaignId, startDt, endDt, pageable); } - - public static String getValueString(Object value) { - if (value instanceof Double || value instanceof Float) { - BigDecimal bd = new BigDecimal(value.toString()); - return bd.stripTrailingZeros().toPlainString(); - } - if (value instanceof Number) { - return value.toString(); - } - return String.valueOf(value); - } } \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java index 5bd265982..656327403 100644 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java @@ -238,7 +238,7 @@ void shouldAssignRootScopeWhenExternalVariableIsMissingFromMetadata() { .extracting(VariableModel::scope) .isEqualTo(Constants.ROOT_GROUP_NAME); } - +//FIXME Faire les devs de feat(rawresponses): keep null values if variable present in last survey... dans lunaticjson converter @Nested @DisplayName("Null cases tests") class NullVariablesTests { diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java index 49a664e5e..9373e2c92 100644 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java @@ -3,7 +3,6 @@ import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -19,6 +18,7 @@ import java.util.Map; import java.util.Set; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.eq; @ExtendWith(MockitoExtension.class) @@ -76,19 +76,66 @@ void getLastSurveyUnitModelsTest() { eq(questionnaireId), setArgumentCaptor.capture() ); - Assertions.assertThat(setArgumentCaptor.getValue()).containsExactlyInAnyOrder( + assertThat(setArgumentCaptor.getValue()).containsExactlyInAnyOrder( "Interrogation1","Interrogation2" ); //Check resulted map key and values content - Assertions.assertThat(resultMap).containsOnlyKeys(interrogationIds); + assertThat(resultMap).containsOnlyKeys(interrogationIds); for(String interrogationId : interrogationIds){ Map surveyUnitModelsOfInterrogationId = resultMap.get(interrogationId); - Assertions.assertThat(surveyUnitModelsOfInterrogationId).containsOnlyKeys(DataState.COLLECTED); - Assertions.assertThat( + assertThat(surveyUnitModelsOfInterrogationId).containsOnlyKeys(DataState.COLLECTED); + assertThat( surveyUnitModelsOfInterrogationId.get(DataState.COLLECTED).getInterrogationId() ).isEqualTo(interrogationId); } } } + + @Nested + @DisplayName("getValueString() util") + class GetValueStringTests { + + @Test + @DisplayName("Double value strips trailing zeros") + void doubleStripsTrailingZeros() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString(1.50)).isEqualTo("1.5"); + } + + @Test + @DisplayName("Float value strips trailing zeros") + void floatStripsTrailingZeros() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString(1.500f)).isEqualTo("1.5"); + } + + @Test + @DisplayName("Integer value returns plain string") + void integerReturnsPlainString() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString(42)).isEqualTo("42"); + } + + @Test + @DisplayName("String value returns same string") + void stringReturnsItself() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString("hello")).isEqualTo("hello"); + } + + @Test + @DisplayName("Null returns 'null' string") + void nullReturnsNullString() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString(null)).isEqualTo("null"); + } + + @Test + @DisplayName("BigDecimal integer-like double has no decimal point") + void bigDecimalIntegerDouble() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString(3.0)).isEqualTo("3"); + } + } } \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index 91164a551..17c5c6470 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -735,53 +735,6 @@ void returnsPage() { } } - @Nested - @DisplayName("getValueString() util") - class GetValueStringTests { - - @Test - @DisplayName("Double value strips trailing zeros") - void doubleStripsTrailingZeros() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString(1.50)).isEqualTo("1.5"); - } - - @Test - @DisplayName("Float value strips trailing zeros") - void floatStripsTrailingZeros() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString(1.500f)).isEqualTo("1.5"); - } - - @Test - @DisplayName("Integer value returns plain string") - void integerReturnsPlainString() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString(42)).isEqualTo("42"); - } - - @Test - @DisplayName("String value returns same string") - void stringReturnsItself() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString("hello")).isEqualTo("hello"); - } - - @Test - @DisplayName("Null returns 'null' string") - void nullReturnsNullString() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString(null)).isEqualTo("null"); - } - - @Test - @DisplayName("BigDecimal integer-like double has no decimal point") - void bigDecimalIntegerDouble() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString(3.0)).isEqualTo("3"); - } - } - //UTILS /** Builds a minimal raw data document with COLLECTED data. */ private LunaticJsonRawDataModel buildRawDataWithCollected(Map collectedData) { From 7b3c7a0fde736c075ef1a07e22a2a83e82307751 Mon Sep 17 00:00:00 2001 From: Alexis Szmundy Date: Mon, 8 Jun 2026 15:41:28 +0200 Subject: [PATCH 09/12] fix: fixed legacy process test --- .../LunaticJsonRawDataConverterTest.java | 57 ++++++++----------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java index 656327403..db8547d5c 100644 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java @@ -9,6 +9,7 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.parser.rawdata.LunaticJsonRawDataPayloadParser; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -17,7 +18,6 @@ import org.junit.jupiter.params.provider.ValueSource; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; import java.time.LocalDateTime; @@ -25,14 +25,14 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -76,40 +76,31 @@ void shouldCreateCollectedAndEditedSurveyUnitsFromLegacyRawData() { ) )); - try (MockedStatic rawResponseConverter = mockStatic(RawResponseRawDataConverter.class)) { - rawResponseConverter - .when(() -> RawResponseRawDataConverter.processCollectedVariableForState(any(), any(), any(), any(), any(), any())) - .thenAnswer(invocation -> { - String state = invocation.getArgument(1); - List destination = invocation.getArgument(4); - - destination.add(VariableModel.builder() - .varId("FIRST_NAME") - .value(state) - .build()); - - return null; - }); - - List result = converter.convertRawData( - QUESTIONNAIRE_ID, - List.of(rawData), - variablesMap - ); + List result = converter.convertRawData( + QUESTIONNAIRE_ID, + List.of(rawData), + variablesMap + ); - assertThat(result).hasSize(2); + assertThat(result).hasSize(2); - assertThat(result) - .extracting(SurveyUnitModel::getState) - .containsExactly(DataState.COLLECTED, DataState.EDITED); + assertThat(result) + .extracting(SurveyUnitModel::getState) + .containsExactly(DataState.COLLECTED, DataState.EDITED); - assertThat(result.get(0).getCollectedVariables()) + for(DataState dataState : Set.of(DataState.COLLECTED, DataState.EDITED)){ + Optional surveyUnitModelOptional = result.stream().filter( + surveyUnitModel -> surveyUnitModel.getState().equals(dataState) + ).findFirst(); + + Assertions.assertThat(surveyUnitModelOptional).isPresent(); + SurveyUnitModel surveyUnitModel = surveyUnitModelOptional.get(); + Assertions.assertThat(surveyUnitModel.getCollectedVariables()) + .extracting(VariableModel::varId) + .containsExactly("FIRST_NAME"); + Assertions.assertThat(surveyUnitModel.getCollectedVariables()) .extracting(VariableModel::value) - .containsExactly("COLLECTED"); - - assertThat(result.get(1).getCollectedVariables()) - .extracting( VariableModel::value) - .containsExactly("EDITED"); + .containsExactly(dataState.equals(DataState.COLLECTED) ? "Alice" : "Alicia"); } } From eb254da4720acc3cdaf8b371f75c8e310dff6a28 Mon Sep 17 00:00:00 2001 From: Alexis Szmundy Date: Mon, 8 Jun 2026 16:23:48 +0200 Subject: [PATCH 10/12] feat(lunaticjsondata): keep null values --- .../converter/rawdata/RawDataConverter.java | 3 ++- .../insee/genesis/domain/utils/DataVerifier.java | 8 ++++++-- .../rest/responses/RawResponseControllerIT.java | 10 ++++++---- .../rawdata/LunaticJsonRawDataConverterTest.java | 16 ++++++---------- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java index aa9da84a7..eb6d5a7b3 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java @@ -160,8 +160,9 @@ private static void fillRawDataMapWithIterations( rawDataMap.put(variableName, rawValuesList); return; } - Map stateMap = (Map) rawDataMap.get(variableName); + Map stateMap = new HashMap<>((Map) rawDataMap.get(variableName)); stateMap.put(dataState.toString(), rawValuesList); + rawDataMap.put(variableName, stateMap); } /** diff --git a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java index e5ba1b6d2..93df174fa 100644 --- a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java +++ b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java @@ -45,7 +45,7 @@ public class DataVerifier { * @param variablesMap VariablesMap containing definitions of each variable */ public static void verifySurveyUnits(List surveyUnitModelsList, VariablesMap variablesMap){ - List surveyUnitModelsListFormatted = new ArrayList<>(); // Created FORCED SU models + List surveyUnitModelsListFormatted = new ArrayList<>(); // Created FORMATTED SU models for(String interrogationId : getInterrogationIds(surveyUnitModelsList)) { // For each id of the list List srcSurveyUnitModelsOfInterrogationId = surveyUnitModelsList.stream().filter(element -> element.getInterrogationId().equals(interrogationId)).toList(); @@ -56,7 +56,7 @@ public static void verifySurveyUnits(List surveyUnitModelsList, collectedVariablesManagement(srcSurveyUnitModelsOfInterrogationId, variablesMap, correctedCollectedVariables); externalVariablesManagement(srcSurveyUnitModelsOfInterrogationId, variablesMap, correctedExternalVariables); - //Create FORCED if any corrected variable + //Create FORMATTED if any corrected variable if(!correctedCollectedVariables.isEmpty() || !correctedExternalVariables.isEmpty()){ SurveyUnitModel newFormattedSurveyUnitModel = createFormattedSurveyUnitModel(surveyUnitModelsList, interrogationId, correctedCollectedVariables, correctedExternalVariables); surveyUnitModelsListFormatted.add(newFormattedSurveyUnitModel); @@ -194,6 +194,10 @@ private static VariableModel verifyVariable( fr.insee.bpm.metadata.model.Variable variableDefinition, DataState dataState ) { + //null values are OK + if(variableModel.value() == null){ + return null; + } if(isParseError(variableModel.value(), variableDefinition.getType(),dataState)){ return VariableModel.builder() .varId(variableModel.varId()) diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java index 3e5c62e68..6b32284a5 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java @@ -508,7 +508,7 @@ void process_raw_response_keep_null_test(boolean isNullSurveyUnitValues) { Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); SurveyUnitDocument savedDocument = savedDocuments.stream().filter( surveyUnitDocument -> - surveyUnitDocument.getInterrogationId().equals("interrogationId")) + surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst())) .toList().getFirst(); Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); @@ -577,7 +577,7 @@ void process_raw_response_null_on_absent_test() { Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); SurveyUnitDocument savedDocument = savedDocuments.stream().filter( surveyUnitDocument -> - surveyUnitDocument.getInterrogationId().equals("interrogationId")) + surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst())) .toList().getFirst(); Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); @@ -891,7 +891,7 @@ void processLunaticJsonRawData_keep_null_test(boolean isNullSurveyUnitValues){ Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); SurveyUnitDocument savedDocument = savedDocuments.stream().filter( surveyUnitDocument -> - surveyUnitDocument.getInterrogationId().equals("interrogationId")) + surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst())) .toList().getFirst(); Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); @@ -961,7 +961,7 @@ void processLunaticJsonRawData_null_on_absent_test(){ Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); SurveyUnitDocument savedDocument = savedDocuments.stream().filter( surveyUnitDocument -> - surveyUnitDocument.getInterrogationId().equals("interrogationId")) + surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst())) .toList().getFirst(); Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); @@ -1269,6 +1269,7 @@ private SurveyUnitDocument getSurveyUnitDocument( alreadyPresentSurveyUnitDocument.setCollectionInstrumentId(collectionInstrumentId); alreadyPresentSurveyUnitDocument.setInterrogationId(interrogationId); alreadyPresentSurveyUnitDocument.setMode(Mode.WEB.getModeName()); + alreadyPresentSurveyUnitDocument.setState("COLLECTED"); alreadyPresentSurveyUnitDocument.setCollectedVariables(new ArrayList<>()); VariableDocument oldVariable = new VariableDocument(); @@ -1284,6 +1285,7 @@ private SurveyUnitDocument getSurveyUnitDocument( oldVariable.setIteration(1); oldVariable.setValue(externalValue); oldVariable.setScope(Constants.ROOT_GROUP_NAME); + alreadyPresentSurveyUnitDocument.getExternalVariables().add(oldVariable); return alreadyPresentSurveyUnitDocument; } diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java index db8547d5c..3230ad697 100644 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java @@ -40,6 +40,7 @@ class LunaticJsonRawDataConverterTest { private static final LocalDateTime DATE_TIME = LocalDateTime.parse("2025-01-01T10:00:00"); private static final String QUESTIONNAIRE_ID = "testQuestionnaire"; + private static final String INTERROGATION_ID = "testInterrogation"; @Mock private SurveyUnitService surveyUnitService; @@ -229,13 +230,12 @@ void shouldAssignRootScopeWhenExternalVariableIsMissingFromMetadata() { .extracting(VariableModel::scope) .isEqualTo(Constants.ROOT_GROUP_NAME); } -//FIXME Faire les devs de feat(rawresponses): keep null values if variable present in last survey... dans lunaticjson converter + @Nested @DisplayName("Null cases tests") class NullVariablesTests { //NullVariablesTests constants - private static final String INTERROGATION_ID = "testInterrogation"; private static final String COLLECTED_VARIABLE_NAME = "VAR1"; private static final String COLLECTED_VARIABLE_VALUE = "test"; private static final String EXTERNAL_VARIABLE_NAME = "EXTVAR1"; @@ -471,6 +471,7 @@ void shouldKeepNull(boolean isNewVariablesPresent){ SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .collectionInstrumentId(QUESTIONNAIRE_ID) .interrogationId(INTERROGATION_ID) + .state(DataState.COLLECTED) .collectedVariables(new ArrayList<>()) .externalVariables(new ArrayList<>()) .build(); @@ -478,7 +479,6 @@ void shouldKeepNull(boolean isNewVariablesPresent){ VariableModel.builder() .varId(COLLECTED_VARIABLE_NAME) .value(null) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(1) .build() @@ -487,7 +487,6 @@ void shouldKeepNull(boolean isNewVariablesPresent){ VariableModel.builder() .varId(EXTERNAL_VARIABLE_NAME) .value(null) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(1) .build() @@ -540,7 +539,6 @@ void shouldKeepNullIteration(boolean isNewVariablesPresent){ VariableModel.builder() .varId(COLLECTED_VARIABLE_NAME) .value(null) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(2) .build() @@ -549,7 +547,6 @@ void shouldKeepNullIteration(boolean isNewVariablesPresent){ VariableModel.builder() .varId(EXTERNAL_VARIABLE_NAME) .value(null) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(2) .build() @@ -603,6 +600,7 @@ private SurveyUnitModel getSurveyUnitModel() { SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .collectionInstrumentId(QUESTIONNAIRE_ID) .interrogationId(INTERROGATION_ID) + .state(DataState.COLLECTED) .collectedVariables(new ArrayList<>()) .externalVariables(new ArrayList<>()) .build(); @@ -611,7 +609,6 @@ private SurveyUnitModel getSurveyUnitModel() { VariableModel.builder() .varId(COLLECTED_VARIABLE_NAME) .value(COLLECTED_VARIABLE_VALUE) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(1) .build() @@ -620,7 +617,6 @@ private SurveyUnitModel getSurveyUnitModel() { VariableModel.builder() .varId(EXTERNAL_VARIABLE_NAME) .value(EXTERNAL_VARIABLE_VALUE) - .state(DataState.COLLECTED) .scope(Constants.ROOT_GROUP_NAME) .iteration(1) .build() @@ -683,9 +679,9 @@ private void assertSecondIterationNull(List surveyUnitModels) { private LunaticJsonRawDataModel rawData(Map data) { LunaticJsonRawDataModel rawData = LunaticJsonRawDataModel.builder() - .questionnaireId("questionnaire-id") + .questionnaireId(QUESTIONNAIRE_ID) .mode(Mode.WEB) - .interrogationId("interrogation-id") + .interrogationId(INTERROGATION_ID) .idUE("survey-unit-id") .recordDate(DATE_TIME) .data(data) From 01e64328bb241b9072ac882bf86b3f4c9684f58e Mon Sep 17 00:00:00 2001 From: Alexis Szmundy Date: Tue, 9 Jun 2026 09:35:26 +0200 Subject: [PATCH 11/12] fix: fixed tests --- .../rawdata/LunaticJsonRawDataConverter.java | 7 +-- .../converter/rawdata/RawDataConverter.java | 11 ++-- .../rawdata/RawResponseRawDataConverter.java | 6 ++- .../genesis/domain/utils/DataVerifier.java | 22 +++----- .../controller/rest/ControllerAccessIT.java | 2 + .../domain/utils/DataVerifierTest.java | 54 +------------------ 6 files changed, 23 insertions(+), 79 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java index eb46f789b..ae02e02af 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java @@ -6,7 +6,7 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.parser.rawdata.LunaticJsonRawDataPayloadParser; -import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -22,8 +22,9 @@ public class LunaticJsonRawDataConverter extends RawDataConverter { private final LunaticJsonRawDataPayloadParser payloadParser; - public LunaticJsonRawDataConverter(SurveyUnitService surveyUnitService, LunaticJsonRawDataPayloadParser lunaticJsonRawDataPayloadParser) { - super(surveyUnitService); + public LunaticJsonRawDataConverter(SurveyUnitApiPort surveyUnitApiPort, + LunaticJsonRawDataPayloadParser lunaticJsonRawDataPayloadParser) { + super(surveyUnitApiPort); this.payloadParser = lunaticJsonRawDataPayloadParser; } diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java index eb6d5a7b3..d2e065041 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java @@ -7,7 +7,7 @@ import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; -import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import fr.insee.genesis.domain.utils.GroupUtils; import fr.insee.genesis.domain.utils.JsonUtils; import jakarta.validation.constraints.NotNull; @@ -28,11 +28,12 @@ @Slf4j @Component public abstract class RawDataConverter { - private final SurveyUnitService surveyUnitService; + @SuppressWarnings("FieldMayBeFinal") //Final would break RawDataConverterTest + private SurveyUnitApiPort surveyUnitApiPort; @Autowired - public RawDataConverter(SurveyUnitService surveyUnitService) { - this.surveyUnitService = surveyUnitService; + public RawDataConverter(SurveyUnitApiPort surveyUnitApiPort) { + this.surveyUnitApiPort = surveyUnitApiPort; } /** @@ -46,7 +47,7 @@ protected Map> getLastSurveyUnitModels( ) { Set interrogationIdsSet = new HashSet<>(interrogationIds); - List surveyUnitModels = surveyUnitService.findLatestByInterrogationIds( + List surveyUnitModels = surveyUnitApiPort.findLatestByInterrogationIds( questionnaireOrCollectionInstrumentId, interrogationIdsSet ); diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java index b6c961dc4..1bd5ba6f2 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java @@ -6,6 +6,7 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser; +import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.modelefiliere.RawResponseDto; import lombok.extern.slf4j.Slf4j; @@ -23,8 +24,9 @@ public class RawResponseRawDataConverter extends RawDataConverter { private final RawResponsePayloadParser rawResponsePayloadParser; - public RawResponseRawDataConverter(SurveyUnitService surveyUnitService, RawResponsePayloadParser rawResponsePayloadParser) { - super(surveyUnitService); + public RawResponseRawDataConverter(SurveyUnitApiPort surveyUnitApiPort, + RawResponsePayloadParser rawResponsePayloadParser) { + super(surveyUnitApiPort); this.rawResponsePayloadParser = rawResponsePayloadParser; } diff --git a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java index 93df174fa..24237d1ae 100644 --- a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java +++ b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java @@ -153,8 +153,7 @@ private static void collectedVariablesManagement(List srcSurvey { VariableModel correctedCollectedVariable = verifyVariable( collectedVariableToVerify.variableModel(), - variablesMap.getVariable(collectedVariableToVerify.variableModel().varId()), - collectedVariableToVerify.dataState() + variablesMap.getVariable(collectedVariableToVerify.variableModel().varId()) ); if(correctedCollectedVariable != null){ @@ -191,14 +190,9 @@ private static void addIteration(VariableModel variableToCheck, private static VariableModel verifyVariable( VariableModel variableModel, - fr.insee.bpm.metadata.model.Variable variableDefinition, - DataState dataState + fr.insee.bpm.metadata.model.Variable variableDefinition ) { - //null values are OK - if(variableModel.value() == null){ - return null; - } - if(isParseError(variableModel.value(), variableDefinition.getType(),dataState)){ + if(isParseError(variableModel.value(), variableDefinition.getType())){ return VariableModel.builder() .varId(variableModel.varId()) .value("") @@ -223,8 +217,7 @@ private static void externalVariablesManagement(List srcSuModel if(variablesMap.hasVariable(externalVariable.varId())) { VariableModel correctedExternalVariable = verifyVariable( externalVariable, - variablesMap.getVariable(externalVariable.varId()), - state + variablesMap.getVariable(externalVariable.varId()) ); if (correctedExternalVariable != null) { correctedExternalVariables.add(correctedExternalVariable); @@ -238,15 +231,12 @@ private static void externalVariablesManagement(List srcSuModel * Use the correct parser and try to parse * @param value value to verify * @param type type of the variable - * @param state state of the data where the variable is contained in * @return true if the value is not conform to the variable type */ - private static boolean isParseError(String value, VariableType type, DataState state){ + private static boolean isParseError(String value, VariableType type){ //Allow null values if(value == null){ - return !(state.equals(DataState.EDITED) - || state.equals(DataState.FORCED) - || state.equals(DataState.FORMATTED)); //Return false if datastate one of those + return false; } switch(type){ case BOOLEAN: diff --git a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java index aeb6345f8..897507762 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java +++ b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java @@ -6,6 +6,7 @@ import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; @@ -54,6 +55,7 @@ class ControllerAccessIT extends IntegrationTestAbstract { private RawResponseInputRepository rawRepository; @MockitoBean private SurveyUnitQualityToolService surveyUnitQualityToolService; + /** * Provides a stream of URIs that are allowed for reader. */ diff --git a/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java b/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java index c2df62269..4ef2d2cbb 100644 --- a/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java +++ b/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java @@ -255,59 +255,7 @@ void shouldCorrectInvalidIterationOnFormattedSurveyUnit() { } @ParameterizedTest - @EnumSource(value = DataState.class, names = {"EDITED", "FORCED", "FORMATTED"}, mode = EnumSource.Mode.EXCLUDE) - void shouldCreateFormattedIfNull(DataState dataState) { - String variableName = "varnull"; - for(VariableType variableType : VariableType.values()){ - if(variableType.equals(VariableType.STRING)){ - continue; //Skip STRING - } - variablesMap.removeVariable(variableName); - - // GIVEN - Variable variableDefinition = new Variable( - variableName, - new MetadataModel().getRootGroup(), - variableType - ); - variablesMap.putVariable(variableDefinition); - - // Add null value - surveyUnits.clear(); - VariableModel collectedVariable1 = VariableModel.builder() - .varId(variableName) - .value(null) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build(); - SurveyUnitModel surveyUnit = SurveyUnitModel.builder() - .interrogationId("UE1100000001") - .collectionInstrumentId("Quest1") - .state(dataState) - .collectedVariables(List.of(collectedVariable1)) - .externalVariables(List.of()) - .build(); - surveyUnits.add(surveyUnit); - - // WHEN - DataVerifier.verifySurveyUnits(surveyUnits, variablesMap); - - // THEN FORMATTED values added - try{ - Assertions.assertTrue(surveyUnits.size() > 1); - SurveyUnitModel formattedUnit = surveyUnits.get(1); - Assertions.assertEquals(DataState.FORMATTED, formattedUnit.getState()); - Assertions.assertEquals(1, formattedUnit.getCollectedVariables().size()); - Assertions.assertEquals("", formattedUnit.getCollectedVariables().getFirst().value()); // Corrected value - }catch (AssertionFailedError afe){ - log.error("Failed on type {}", variableType); - throw afe; - } - } - } - - @ParameterizedTest - @EnumSource(value = DataState.class, names = {"EDITED", "FORCED", "FORMATTED"}) + @EnumSource(value = DataState.class) void shouldNotCreateFormattedIfNull(DataState dataState) { String variableName = "varnull"; for(VariableType variableType : VariableType.values()){ From ed1f4dd1574c200cdaa095f72164e50ea2f29641 Mon Sep 17 00:00:00 2001 From: Hajarel-moukh Date: Tue, 9 Jun 2026 14:24:34 +0200 Subject: [PATCH 12/12] add bpm import --- src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java index 24237d1ae..e2667740f 100644 --- a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java +++ b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java @@ -1,5 +1,6 @@ package fr.insee.genesis.domain.utils; +import fr.insee.bpm.metadata.model.Variable; import fr.insee.bpm.metadata.model.VariableType; import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.Constants; @@ -190,7 +191,7 @@ private static void addIteration(VariableModel variableToCheck, private static VariableModel verifyVariable( VariableModel variableModel, - fr.insee.bpm.metadata.model.Variable variableDefinition + Variable variableDefinition ) { if(isParseError(variableModel.value(), variableDefinition.getType())){ return VariableModel.builder()