diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index eba3557f4..959d71543 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -286,7 +286,7 @@ public ResponseEntity findResponsesByInterrogationAndCollectionInstrumen @PreAuthorize("hasRole('ADMIN')") public ResponseEntity> getLatestByInterrogationAndCollectionInstrument(@RequestParam("interrogationId") String interrogationId, @RequestParam("collectionInstrumentId") String collectionInstrumentId) { - List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); + List responses = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId); return ResponseEntity.ok(responses); } @@ -300,7 +300,7 @@ public ResponseEntity> getLatestByInterrogationAndCollecti public ResponseEntity getLatestByInterrogationOneObject(@RequestParam("interrogationId") String interrogationId, @RequestParam("collectionInstrumentId") String collectionInstrumentId, @RequestParam("mode") Mode mode) { - List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); + List responses = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId); List outputVariables = new ArrayList<>(); List outputExternalVariables = new ArrayList<>(); RawResponseDto.QuestionnaireStateEnum questionnaireState = null; @@ -365,7 +365,7 @@ public ResponseEntity> getLatestForInterrogationLi List results = new ArrayList<>(); List modes = surveyUnitService.findModesByCollectionInstrumentId(collectionInstrumentId); interrogationIds.forEach(interrogationId -> { - List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List responses = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( interrogationId.getInterrogationId(), collectionInstrumentId ); modes.forEach(mode -> { diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java index 66f99ff96..ae02e02af 100644 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java @@ -1,17 +1,12 @@ package fr.insee.genesis.domain.converter.rawdata; -import fr.insee.bpm.metadata.model.Variable; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.parser.rawdata.LunaticJsonRawDataPayloadParser; -import fr.insee.genesis.domain.utils.GroupUtils; -import fr.insee.genesis.domain.utils.JsonUtils; -import lombok.RequiredArgsConstructor; +import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -19,33 +14,58 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Component @Slf4j -@RequiredArgsConstructor -public class LunaticJsonRawDataConverter { +public class LunaticJsonRawDataConverter extends RawDataConverter { private final LunaticJsonRawDataPayloadParser payloadParser; + public LunaticJsonRawDataConverter(SurveyUnitApiPort surveyUnitApiPort, + LunaticJsonRawDataPayloadParser lunaticJsonRawDataPayloadParser) { + super(surveyUnitApiPort); + this.payloadParser = lunaticJsonRawDataPayloadParser; + } + public List convertRawData( + String questionnaireId, List rawDataList, VariablesMap variablesMap ) { - return convertRawDataAndCollectEmptyModels(rawDataList, variablesMap, new ArrayList<>()); + return convertRawDataAndCollectEmptyModels( + questionnaireId, + rawDataList, + variablesMap, + new ArrayList<>() + ); } public List convertRawDataAndCollectEmptyModels( + String questionnaireId, List rawDataList, VariablesMap variablesMap, List emptySurveyUnitModels ) { List surveyUnitModels = new ArrayList<>(); + Map> lastSurveyUnitModelsByInterrogationIdAndState = + getLastSurveyUnitModels( + questionnaireId, + rawDataList.stream().map(LunaticJsonRawDataModel::interrogationId).collect(Collectors.toList()) + ); + for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) { for (LunaticJsonRawDataModel rawData : rawDataList) { RawDataModelType rawDataModelType = getRawDataModelType(rawData); + SurveyUnitModel lastSurveyUnitModelForDataState = null; + if(lastSurveyUnitModelsByInterrogationIdAndState.containsKey(rawData.interrogationId())){ + lastSurveyUnitModelForDataState = lastSurveyUnitModelsByInterrogationIdAndState + .get(rawData.interrogationId()) + .get(dataState); + } - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + SurveyUnitModel newSurveyUnitModel = SurveyUnitModel.builder() .collectionInstrumentId(rawData.questionnaireId()) .mode(rawData.mode()) .interrogationId(rawData.interrogationId()) @@ -60,25 +80,39 @@ public List convertRawDataAndCollectEmptyModels( .externalVariables(new ArrayList<>()) .build(); - convertRawDataCollectedVariables(rawData, surveyUnitModel, dataState, rawDataModelType, variablesMap); + convertCollectedVariables( + rawData.data(), + rawData.interrogationId(), + lastSurveyUnitModelForDataState, + newSurveyUnitModel, + dataState, + rawDataModelType, + variablesMap + ); if (dataState == DataState.COLLECTED) { - convertRawDataExternalVariables(rawData, surveyUnitModel, rawDataModelType, variablesMap); + convertExternalVariables( + rawData.data(), + lastSurveyUnitModelForDataState, + newSurveyUnitModel, + rawDataModelType, + variablesMap + ); } - boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() - && surveyUnitModel.getExternalVariables().isEmpty(); + boolean hasNoVariable = newSurveyUnitModel.getCollectedVariables().isEmpty() + && newSurveyUnitModel.getExternalVariables().isEmpty(); if (hasNoVariable) { - if (surveyUnitModel.getState() == DataState.COLLECTED) { + if (newSurveyUnitModel.getState() == DataState.COLLECTED) { log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawData.interrogationId()); } - emptySurveyUnitModels.add(surveyUnitModel); + emptySurveyUnitModels.add(newSurveyUnitModel); continue; } - surveyUnitModels.add(surveyUnitModel); + surveyUnitModels.add(newSurveyUnitModel); } } @@ -90,127 +124,4 @@ private static RawDataModelType getRawDataModelType(LunaticJsonRawDataModel rawD ? RawDataModelType.FILIERE : RawDataModelType.LEGACY; } - - private void convertRawDataCollectedVariables( - LunaticJsonRawDataModel srcRawData, - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - RawDataModelType rawDataModelType, - VariablesMap variablesMap - ) { - Map dataMap = srcRawData.data(); - if (rawDataModelType == RawDataModelType.FILIERE) { - dataMap = JsonUtils.asMap(dataMap.get("data")); - } - - Map collectedMap = JsonUtils.asMap(dataMap.get("COLLECTED")); - - if (collectedMap == null || collectedMap.isEmpty()) { - if (dataState == DataState.COLLECTED) { - log.warn("No collected data for interrogation {}", srcRawData.interrogationId()); - } - return; - } - - String stateKey = dataState.toString(); - List destination = dstSurveyUnitModel.getCollectedVariables(); - - for (Map.Entry collectedVariable : collectedMap.entrySet()) { - RawResponseConverter.processCollectedVariable( - collectedVariable, - stateKey, - variablesMap, - dstSurveyUnitModel, - destination - ); - } - } - - - private static void convertRawDataExternalVariables( - LunaticJsonRawDataModel srcRawData, - SurveyUnitModel dstSurveyUnitModel, - RawDataModelType rawDataModelType, - VariablesMap variablesMap - ) { - Map dataMap = srcRawData.data(); - if (rawDataModelType == RawDataModelType.FILIERE) { - dataMap = JsonUtils.asMap(dataMap.get("data")); - } - - Map externalMap = JsonUtils.asMap(dataMap.get("EXTERNAL")); - if (externalMap != null && !externalMap.isEmpty()) { - convertToExternalVar(dstSurveyUnitModel, variablesMap, externalMap); - } - } - - private static void convertToExternalVar( - SurveyUnitModel dstSurveyUnitModel, - VariablesMap variablesMap, - Map externalMap - ) { - for (Map.Entry externalVariableEntry : externalMap.entrySet()) { - Object valueObject = externalVariableEntry.getValue(); - - if (valueObject instanceof List) { - convertListVar(valueObject, externalVariableEntry, variablesMap, dstSurveyUnitModel.getExternalVariables()); - continue; - } - - if (valueObject != null) { - convertOneVar( - externalVariableEntry, - valueObject.toString(), - variablesMap, - 1, - dstSurveyUnitModel.getExternalVariables() - ); - } - } - } - - private static void convertListVar( - Object valuesForState, - Map.Entry collectedVariable, - VariablesMap variablesMap, - List destination - ) { - List values = JsonUtils.asStringList(valuesForState); - if (!values.isEmpty()) { - int iteration = 1; - for (String value : values) { - if (value != null && !value.isEmpty()) { - convertOneVar(collectedVariable, value, variablesMap, iteration, destination); - } - iteration++; - } - } - } - - private static void convertOneVar( - Map.Entry variableEntry, - String value, - VariablesMap variablesMap, - int iteration, - List destination - ) { - VariableModel variableModel = VariableModel.builder() - .varId(variableEntry.getKey()) - .value(value) - .scope(getIdLoop(variablesMap, variableEntry.getKey())) - .iteration(iteration) - .parentId(GroupUtils.getParentGroupName(variableEntry.getKey(), variablesMap)) - .build(); - - destination.add(variableModel); - } - - private static String getIdLoop(VariablesMap variablesMap, String variableName) { - Variable variable = variablesMap.getVariable(variableName); - if (variable == null) { - log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); - return Constants.ROOT_GROUP_NAME; - } - return variable.getGroupName(); - } } diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java new file mode 100644 index 000000000..d2e065041 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java @@ -0,0 +1,553 @@ +package fr.insee.genesis.domain.converter.rawdata; + +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; +import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.utils.GroupUtils; +import fr.insee.genesis.domain.utils.JsonUtils; +import jakarta.validation.constraints.NotNull; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.lang.Nullable; +import org.springframework.stereotype.Component; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +@Component +public abstract class RawDataConverter { + @SuppressWarnings("FieldMayBeFinal") //Final would break RawDataConverterTest + private SurveyUnitApiPort surveyUnitApiPort; + + @Autowired + public RawDataConverter(SurveyUnitApiPort surveyUnitApiPort) { + this.surveyUnitApiPort = surveyUnitApiPort; + } + + /** + * @param questionnaireOrCollectionInstrumentId Questionnaire/Collection instrument id + * @param interrogationIds list of interrogation ids + * @return a Map containing latest survey unit models for each interrogation ids + */ + protected Map> getLastSurveyUnitModels( + String questionnaireOrCollectionInstrumentId, + List interrogationIds + ) { + Set interrogationIdsSet = new HashSet<>(interrogationIds); + + List surveyUnitModels = surveyUnitApiPort.findLatestByInterrogationIds( + questionnaireOrCollectionInstrumentId, + interrogationIdsSet + ); + + Map> surveyUnitModelsByInterrogationIdAndState = new HashMap<>(); + + for (String interrogationId : interrogationIdsSet){ + List surveyUnitModelsForInterrogationId = surveyUnitModels.stream().filter( + surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(interrogationId) + ).toList(); + if(surveyUnitModelsForInterrogationId.isEmpty()){ + continue; + } + addSurveyUnitsOfInterrogationByState( + interrogationId, + surveyUnitModelsForInterrogationId, + surveyUnitModelsByInterrogationIdAndState + ); + } + + return surveyUnitModelsByInterrogationIdAndState; + } + + private static void addSurveyUnitsOfInterrogationByState( + String interrogationId, + List surveyUnitModelsForInterrogationId, + Map> surveyUnitModelsByInterrogationIdAndState + ) { + surveyUnitModelsByInterrogationIdAndState.put(interrogationId, new HashMap<>()); + for(SurveyUnitModel surveyUnitOfInterrogation : surveyUnitModelsForInterrogationId){ + if(surveyUnitOfInterrogation.getState() == null){ + continue; + } + surveyUnitModelsByInterrogationIdAndState.get(interrogationId).put( + surveyUnitOfInterrogation.getState(), + surveyUnitOfInterrogation + ); + } + } + + /** + *

Adds missing variables/iterations to raw data map based on last survey unit in database

+ *

e.g. raw: var1 absent AND response: var1 present -> var1 added with null value

+ * + * @param rawDataMap dataMap (COLLECTED or EXTERNAL) + * @param dataState null if EXTERNAL + */ + private static void fillRawDataMapWithAbsentVariables( + @NotNull SurveyUnitModel lastSurveyUnit, + Map rawDataMap, + @Nullable DataState dataState + ) { + List lastSurveyUnitVariables = dataState == null ? + lastSurveyUnit.getExternalVariables() : lastSurveyUnit.getCollectedVariables(); + + //Extract variable names + Set lastSurveyUnitVariablesNames = lastSurveyUnitVariables.stream() + .map(VariableModel::varId).collect(Collectors.toSet()); + + for(String lastSurveyUnitVariableName : lastSurveyUnitVariablesNames){ + List variableModelsForSameVariable = lastSurveyUnitVariables.stream().filter( + variableAlreadyPresent -> + variableAlreadyPresent.varId().equals(lastSurveyUnitVariableName) + ).toList(); + + //If multiple iterations + if(variableModelsForSameVariable.size() > 1){ + fillRawDataMapWithIterations( + variableModelsForSameVariable, + rawDataMap, + dataState + ); + continue; + } + addNullVariableToRawDataMapIfAbsent(lastSurveyUnitVariableName, rawDataMap); + } + } + + /** + * Checks each iteration of a last response variable and adds null to raw data if iteration is absent + * @param alreadyPresentVariableIterations Variables models for same variable in last response + * @param rawDataMap data map (COLLECTED or EXTERNAL) + * @param dataState null if EXTERNAL + */ + @SuppressWarnings("unchecked") + private static void fillRawDataMapWithIterations( + List alreadyPresentVariableIterations, + Map rawDataMap, + @Nullable DataState dataState + ){ + if(alreadyPresentVariableIterations == null || alreadyPresentVariableIterations.isEmpty()){ + return; + } + String variableName = alreadyPresentVariableIterations.getFirst().varId(); + + int maxResponseIteration = alreadyPresentVariableIterations.stream() + .map(VariableModel::iteration) + .max(Integer::compareTo).orElse(0); + + List rawValuesList = getRawDataListForVariable( + variableName, + rawDataMap, + dataState + ); + + //Add null element at end of raw list until max iteration + fillListUntilMaxIteration(rawValuesList, maxResponseIteration); + + //Replace list in raw data map + if(dataState == null){ + rawDataMap.put(variableName, rawValuesList); + return; + } + Map stateMap = new HashMap<>((Map) rawDataMap.get(variableName)); + stateMap.put(dataState.toString(), rawValuesList); + rawDataMap.put(variableName, stateMap); + } + + /** + * @param variableName Name of the variable contained in last response + * @param rawDataMap data map (COLLECTED or EXTERNAL) + * @param dataState null if EXTERNAL + * @return The list of values to put in raw data + */ + @SuppressWarnings("unchecked") + private static List getRawDataListForVariable( + String variableName, + Map rawDataMap, + DataState dataState + ) { + //Instantiate list in rawDataMap if variable exists in response but not in raw + if(!rawDataMap.containsKey(variableName)){ + return instantiateEmptyListInRawDataMap(variableName, rawDataMap, dataState); + } + + if(dataState == null){ + return rawDataMap.get(variableName) instanceof List ? + (List) rawDataMap.get(variableName) + //If single value in raw but multiple in response, return a list with the single raw value + : instantiateListInRawDataMapWithSingleValue(variableName, rawDataMap, null); + } + Map stateMap = (Map) rawDataMap.get(variableName); + return stateMap.get(dataState.toString()) instanceof List ? + (List) stateMap.get(dataState.toString()) + : instantiateListInRawDataMapWithSingleValue(variableName, rawDataMap, dataState); + } + + /** + * @param variableName Name of the variable to add in map + * @param rawDataMap COLLECTED or EXTERNAL map to fill + * @param dataState null if in EXTERNAL map, COLLECTED if not + * @return the instantiated list + */ + private static List instantiateEmptyListInRawDataMap( + String variableName, + Map rawDataMap, + @Nullable DataState dataState + ) { + List valueList = new ArrayList<>(); + if(dataState == null){ + rawDataMap.put(variableName, valueList); + return valueList; + } + Map statesMap = new HashMap<>(); + statesMap.put(dataState.toString(), valueList); + rawDataMap.put(variableName, statesMap); + + return valueList; + } + + @SuppressWarnings("unchecked") + private static List instantiateListInRawDataMapWithSingleValue( + String variableName, + Map rawDataMap, + DataState dataState + ) { + List singleValueList = new ArrayList<>(1); + if(dataState == null){ + singleValueList.add(rawDataMap.get(variableName)); + return singleValueList; + } + Map stateMap = (Map) rawDataMap.get(variableName); + singleValueList.add(stateMap.get(dataState.toString())); + return singleValueList; + } + + /** + * Fills the list of values with null until max iteration + * @param rawValuesList raw data values for a variable + * @param maxResponseIteration max iteration in last survey unit for said variable + */ + private static void fillListUntilMaxIteration(List rawValuesList, int maxResponseIteration) { + for(int i = 1; i <= maxResponseIteration; i++){ + if(rawValuesList.size() >= i){ + continue; + } + rawValuesList.addLast(null); + } + } + + private static void addNullVariableToRawDataMapIfAbsent( + String variableName, + Map rawDataMap + ) { + if (!rawDataMap.containsKey(variableName)){ + rawDataMap.put(variableName, null); + } + } + + protected void convertCollectedVariables( + Map dataOrPayloadMap, + String interrogationId, + @Nullable SurveyUnitModel lastSurveyUnitModel, + SurveyUnitModel dstSurveyUnitModel, + DataState dataState, + RawDataModelType rawDataModelType, + VariablesMap variablesMap + ) { + //Get data map from payload + Map dataMap = dataOrPayloadMap; + if (rawDataModelType == RawDataModelType.FILIERE) { + dataMap = JsonUtils.asMap(dataOrPayloadMap.get("data")); + } + Map collectedMap = new HashMap<>(JsonUtils.asMap(dataMap.get("COLLECTED"))); + + if(lastSurveyUnitModel != null && lastSurveyUnitModel.getState().equals(dataState)) { + fillRawDataMapWithAbsentVariables(lastSurveyUnitModel, collectedMap, dataState); + } + + if (collectedMap.isEmpty()) { + if (dataState == DataState.COLLECTED) { + log.warn("No collected data for interrogation {}", interrogationId); + } + return; + } + + String stateKey = dataState.toString(); + List collectedVariables = dstSurveyUnitModel.getCollectedVariables(); + + for (Map.Entry entry : collectedMap.entrySet()) { + processCollectedVariableForState( + entry, + stateKey, + variablesMap, + lastSurveyUnitModel, + dstSurveyUnitModel, + collectedVariables + ); + } + } + + static void processCollectedVariableForState( + Map.Entry entry, + String stateKey, + VariablesMap variablesMap, + @Nullable SurveyUnitModel lastSurveyUnitModel, + SurveyUnitModel dstSurveyUnitModel, + List variableModelList + ) { + if (Constants.PAIRWISES.equals(entry.getKey())) { + handlePairwiseCollectedVariable(entry, DataState.valueOf(stateKey), variablesMap, dstSurveyUnitModel); + return; + } + + Map variableStates = JsonUtils.asMap(entry.getValue()); + //If variable value absent for state or null value + if (variableStates == null || variableStates.get(stateKey) == null) { + convertNullVar(entry.getKey(), lastSurveyUnitModel, 1, variablesMap, variableModelList, true); + return; + } + + Object value = variableStates.get(stateKey); + if (value instanceof List list) { + convertListVar(list, lastSurveyUnitModel, entry, variablesMap, variableModelList, true); + return; + } + convertOneVar(entry.getKey(), getValueString(value), variablesMap, 1, variableModelList); + } + + @SuppressWarnings("unchecked") + static void handlePairwiseCollectedVariable( + Map.Entry collectedVariable, + DataState dataState, + VariablesMap variablesMap, + SurveyUnitModel dstSurveyUnitModel + ) { + Object value = getValueForState(collectedVariable, dataState.toString()); + + if (isInvalidPairwiseVariable(value, variablesMap)) { + return; + } + + List individuals = (List) value; + String groupName = variablesMap.getVariable(Constants.PAIRWISE_PREFIX + 1).getGroupName(); + + for (int individualIndex = 0; individualIndex < individuals.size(); individualIndex++) { + List individualLinks = (List) individuals.get(individualIndex); + + for (int linkIndex = 1; linkIndex < Constants.MAX_LINKS_ALLOWED; linkIndex++) { + dstSurveyUnitModel.getCollectedVariables().add( + buildPairwiseVariable(individualLinks, linkIndex, individualIndex + 1, groupName) + ); + } + } + } + + private static VariableModel buildPairwiseVariable( + List individualLinks, + int linkIndex, + int iteration, + String groupName + ) { + String value = Constants.NO_PAIRWISE_VALUE; + + if (linkIndex <= individualLinks.size()) { + String v = individualLinks.get(linkIndex - 1); + value = (v == null || v.isBlank()) ? Constants.SAME_AXIS_VALUE : v; + } + + return VariableModel.builder() + .varId(Constants.PAIRWISE_PREFIX + linkIndex) + .value(value) + .scope(groupName) + .iteration(iteration) + .parentId(Constants.ROOT_GROUP_NAME) + .build(); + } + + private static Object getValueForState( + Map.Entry collectedVariable, + String stateKey + ) { + Map states = JsonUtils.asMap(collectedVariable.getValue()); + return states != null ? states.get(stateKey) : null; + } + + private static boolean isInvalidPairwiseVariable(Object value, VariablesMap variablesMap) { + return !(value instanceof List) || !variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1); + } + + private static void convertNullVar( + String variableName, + @Nullable SurveyUnitModel lastSurveyUnitModel, + int iteration, + VariablesMap variablesMap, + List destination, + boolean isCollected //true if in collected variables, false if in external + ) { + //Do nothing if last response is null or variable is not present in last response + if (lastSurveyUnitModel == null || isVariableAbsentInSurveyUnitModel( + variableName, iteration, lastSurveyUnitModel, isCollected + )) { + return; + } + convertOneVar(variableName, null, variablesMap, iteration, destination); + } + + private static boolean isVariableAbsentInSurveyUnitModel( + String variableName, + int iteration, + SurveyUnitModel lastSurveyUnitModel, + boolean isCollected + ) { + //Check in collected variables + if(isCollected){ + return lastSurveyUnitModel.getCollectedVariables().stream().noneMatch( + variableModel -> variableModel.varId().equals(variableName) + && variableModel.iteration().equals(iteration) + ); + } + //Check in external variables + return lastSurveyUnitModel.getExternalVariables().stream().noneMatch( + variableModel -> variableModel.varId().equals(variableName) + && variableModel.iteration().equals(iteration) + ); + } + + private static void convertOneVar( + String variableName, + String value, + VariablesMap variablesMap, + int iteration, + List destination + ) { + VariableModel variableModel = VariableModel.builder() + .varId(variableName) + .value(value) + .scope(getIdLoop(variablesMap, variableName)) + .iteration(iteration) + .parentId(GroupUtils.getParentGroupName(variableName, variablesMap)) + .build(); + + destination.add(variableModel); + } + + private static String getIdLoop(VariablesMap variablesMap, String variableName) { + Variable variable = variablesMap.getVariable(variableName); + if (variable == null) { + log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); + return Constants.ROOT_GROUP_NAME; + } + return variable.getGroupName(); + } + + private static void convertListVar( + Object valuesForState, + @Nullable SurveyUnitModel lastSurveyUnit, + Map.Entry variableEntry, + VariablesMap variablesMap, + List destination, + boolean isCollected + ) { + List values = JsonUtils.asStringList(valuesForState); + + if (!values.isEmpty()) { + int iteration = 1; + for (String value : values) { + if(value == null || value.isEmpty()){ + convertNullVar( + variableEntry.getKey(), + lastSurveyUnit, + iteration, + variablesMap, + destination, + isCollected); + iteration++; + continue; + } + convertOneVar(variableEntry.getKey(), value, variablesMap, iteration, destination); + iteration++; + } + } + } + + protected void convertExternalVariables( + Map dataOrPayloadMap, + @Nullable SurveyUnitModel lastSurveyUnitModel, + SurveyUnitModel dstSurveyUnitModel, + RawDataModelType rawDataModelType, + VariablesMap variablesMap + ) { + //Get data map from payload if filiere model + Map dataMap = dataOrPayloadMap; + if (rawDataModelType == RawDataModelType.FILIERE) { + dataMap = JsonUtils.asMap(dataOrPayloadMap.get("data")); + } + Map externalMap = null; + if(dataMap.containsKey("EXTERNAL")) { + externalMap = new HashMap<>(JsonUtils.asMap(dataMap.get("EXTERNAL"))); + } + + if(lastSurveyUnitModel != null){ + fillRawDataMapWithAbsentVariables(lastSurveyUnitModel, externalMap, null); + } + if(externalMap == null){ + return; + } + + for (Map.Entry externalVariableEntry : externalMap.entrySet()) { + Object valueObject = externalVariableEntry.getValue(); + + if (valueObject == null) { + convertNullVar( + externalVariableEntry.getKey(), + lastSurveyUnitModel, + 1, + variablesMap, + dstSurveyUnitModel.getExternalVariables(), + false); + continue; + } + if (valueObject instanceof List) { + convertListVar( + valueObject, + lastSurveyUnitModel, + externalVariableEntry, + variablesMap, + dstSurveyUnitModel.getExternalVariables(), + false + ); + continue; + } + convertOneVar( + externalVariableEntry.getKey(), + valueObject.toString(), + variablesMap, + 1, + dstSurveyUnitModel.getExternalVariables() + ); + } + } + + static String getValueString(Object value) { + if (value instanceof Double || value instanceof Float) { + BigDecimal bd = new BigDecimal(value.toString()); + return bd.stripTrailingZeros().toPlainString(); + } + if (value instanceof Number) { + return value.toString(); + } + return String.valueOf(value); + } +} diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverter.java deleted file mode 100644 index ce9dec62e..000000000 --- a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverter.java +++ /dev/null @@ -1,303 +0,0 @@ -package fr.insee.genesis.domain.converter.rawdata; - -import fr.insee.bpm.metadata.model.Variable; -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; -import fr.insee.genesis.domain.model.surveyunit.DataState; -import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.VariableModel; -import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; -import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser; -import fr.insee.genesis.domain.utils.GroupUtils; -import fr.insee.genesis.domain.utils.JsonUtils; -import fr.insee.modelefiliere.RawResponseDto; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService.getValueString; - -@Component -@Slf4j -@RequiredArgsConstructor -public class RawResponseConverter { - - private final RawResponsePayloadParser rawResponsePayloadParser; - - public List convertRawResponse( - List rawResponseModels, - VariablesMap variablesMap - ) { - return convertRawResponseAndCollectEmptyModels(rawResponseModels, variablesMap, new ArrayList<>()); - } - - public List convertRawResponseAndCollectEmptyModels( - List rawResponseModels, - VariablesMap variablesMap, - List emptySurveyUnitModels - ) { - List surveyUnitModels = new ArrayList<>(); - - for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) { - for (RawResponseModel rawResponseModel : rawResponseModels) { - SurveyUnitModel surveyUnitModel = buildSurveyUnitModel(rawResponseModel, dataState); - - convertCollectedVariables(rawResponseModel, surveyUnitModel, dataState, variablesMap); - - if (dataState == DataState.COLLECTED) { - convertExternalVariables(rawResponseModel, surveyUnitModel, variablesMap); - } - - boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() - && surveyUnitModel.getExternalVariables().isEmpty(); - - if (hasNoVariable) { - if (surveyUnitModel.getState() == DataState.COLLECTED) { - log.warn( - "No collected or external variable for interrogation {}, raw data is ignored.", - rawResponseModel.interrogationId() - ); - } - emptySurveyUnitModels.add(surveyUnitModel); - continue; - } - - surveyUnitModels.add(surveyUnitModel); - } - } - - return surveyUnitModels; - } - - private SurveyUnitModel buildSurveyUnitModel(RawResponseModel rawResponseModel, DataState dataState) { - String questionnaireStateString = - rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState"); - - RawResponseDto.QuestionnaireStateEnum questionnaireStateEnum = null; - try { - questionnaireStateEnum = RawResponseDto.QuestionnaireStateEnum.valueOf(questionnaireStateString); - } catch (IllegalArgumentException _) { - log.warn("'{}' is not a valid questionnaire state according to filiere model", questionnaireStateString); - } catch (NullPointerException _) { - //Nothing to do - } - - return SurveyUnitModel.builder() - .collectionInstrumentId(rawResponseModel.collectionInstrumentId()) - .majorModelVersion(rawResponsePayloadParser.getStringField(rawResponseModel, "majorModelVersion")) - .mode(rawResponseModel.mode()) - .interrogationId(rawResponseModel.interrogationId()) - .usualSurveyUnitId(rawResponsePayloadParser.getStringField(rawResponseModel, "usualSurveyUnitId")) - .questionnaireState(questionnaireStateEnum) - .validationDate(rawResponsePayloadParser.getValidationDate(rawResponseModel)) - .isCapturedIndirectly(rawResponsePayloadParser.getIsCapturedIndirectly(rawResponseModel)) - .state(dataState) - .fileDate(rawResponseModel.recordDate()) - .recordDate(Instant.now()) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - } - - private void convertCollectedVariables( - RawResponseModel rawResponseModel, - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - VariablesMap variablesMap - ) { - Map dataMap = rawResponseModel.payload(); - dataMap = JsonUtils.asMap(dataMap.get("data")); - Map collectedMap = JsonUtils.asMap(dataMap.get("COLLECTED")); - - if (collectedMap == null || collectedMap.isEmpty()) { - if (dataState == DataState.COLLECTED) { - log.warn("No collected data for interrogation {}", rawResponseModel.interrogationId()); - } - return; - } - - String stateKey = dataState.toString(); - List collectedVariables = dstSurveyUnitModel.getCollectedVariables(); - - for (Map.Entry entry : collectedMap.entrySet()) { - processCollectedVariable(entry, stateKey, variablesMap, dstSurveyUnitModel, collectedVariables); - } - } - - public static void processCollectedVariable( - Map.Entry entry, - String stateKey, - VariablesMap variablesMap, - SurveyUnitModel dstSurveyUnitModel, - List variableModelList - ) { - if (Constants.PAIRWISES.equals(entry.getKey())) { - handlePairwiseCollectedVariable(entry, DataState.valueOf(stateKey), variablesMap, dstSurveyUnitModel); - return; - } - - Map states = JsonUtils.asMap(entry.getValue()); - if (states == null) { - return; - } - - Object value = states.get(stateKey); - if (value == null) { - return; - } - - if (value instanceof List list) { - convertListVar(list, entry, variablesMap, variableModelList); - } else { - convertOneVar(entry, getValueString(value), variablesMap, 1, variableModelList); - } - } - - private static void convertListVar( - Object valuesForState, - Map.Entry variableEntry, - VariablesMap variablesMap, - List destination - ) { - List values = JsonUtils.asStringList(valuesForState); - - if (!values.isEmpty()) { - int iteration = 1; - for (String value : values) { - if (value != null && !value.isEmpty()) { - convertOneVar(variableEntry, value, variablesMap, iteration, destination); - } - iteration++; - } - } - } - - private static void convertOneVar( - Map.Entry variableEntry, - String value, - VariablesMap variablesMap, - int iteration, - List destination - ) { - VariableModel variableModel = VariableModel.builder() - .varId(variableEntry.getKey()) - .value(value) - .scope(getIdLoop(variablesMap, variableEntry.getKey())) - .iteration(iteration) - .parentId(GroupUtils.getParentGroupName(variableEntry.getKey(), variablesMap)) - .build(); - - destination.add(variableModel); - } - - private static String getIdLoop(VariablesMap variablesMap, String variableName) { - Variable variable = variablesMap.getVariable(variableName); - if (variable == null) { - log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); - return Constants.ROOT_GROUP_NAME; - } - return variable.getGroupName(); - } - - private void convertExternalVariables( - RawResponseModel rawResponseModel, - SurveyUnitModel dstSurveyUnitModel, - VariablesMap variablesMap - ) { - Map dataMap = rawResponseModel.payload(); - dataMap = JsonUtils.asMap(dataMap.get("data")); - Map externalMap = JsonUtils.asMap(dataMap.get("EXTERNAL")); - - if (externalMap != null && !externalMap.isEmpty()) { - for (Map.Entry externalVariableEntry : externalMap.entrySet()) { - Object valueObject = externalVariableEntry.getValue(); - - if (valueObject instanceof List) { - convertListVar( - valueObject, - externalVariableEntry, - variablesMap, - dstSurveyUnitModel.getExternalVariables() - ); - continue; - } - - if (valueObject != null) { - convertOneVar( - externalVariableEntry, - valueObject.toString(), - variablesMap, - 1, - dstSurveyUnitModel.getExternalVariables() - ); - } - } - } - } - - @SuppressWarnings("unchecked") - static void handlePairwiseCollectedVariable( - Map.Entry collectedVariable, - DataState dataState, - VariablesMap variablesMap, - SurveyUnitModel dstSurveyUnitModel - ) { - Object value = getValueForState(collectedVariable, dataState.toString()); - - if (isInvalidPairwiseVariable(value, variablesMap)) { - return; - } - - List individuals = (List) value; - String groupName = variablesMap.getVariable(Constants.PAIRWISE_PREFIX + 1).getGroupName(); - - for (int individualIndex = 0; individualIndex < individuals.size(); individualIndex++) { - List individualLinks = (List) individuals.get(individualIndex); - - for (int linkIndex = 1; linkIndex < Constants.MAX_LINKS_ALLOWED; linkIndex++) { - dstSurveyUnitModel.getCollectedVariables().add( - buildPairwiseVariable(individualLinks, linkIndex, individualIndex + 1, groupName) - ); - } - } - } - - private static VariableModel buildPairwiseVariable( - List individualLinks, - int linkIndex, - int iteration, - String groupName - ) { - String value = Constants.NO_PAIRWISE_VALUE; - - if (linkIndex <= individualLinks.size()) { - String v = individualLinks.get(linkIndex - 1); - value = (v == null || v.isBlank()) ? Constants.SAME_AXIS_VALUE : v; - } - - return VariableModel.builder() - .varId(Constants.PAIRWISE_PREFIX + linkIndex) - .value(value) - .scope(groupName) - .iteration(iteration) - .parentId(Constants.ROOT_GROUP_NAME) - .build(); - } - - private static Object getValueForState( - Map.Entry collectedVariable, - String stateKey - ) { - Map states = JsonUtils.asMap(collectedVariable.getValue()); - return states != null ? states.get(stateKey) : null; - } - - private static boolean isInvalidPairwiseVariable(Object value, VariablesMap variablesMap) { - return !(value instanceof List) || !variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1); - } -} diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java new file mode 100644 index 000000000..1bd5ba6f2 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java @@ -0,0 +1,150 @@ +package fr.insee.genesis.domain.converter.rawdata; + +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; +import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser; +import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.modelefiliere.RawResponseDto; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Component +@Slf4j +public class RawResponseRawDataConverter extends RawDataConverter { + + private final RawResponsePayloadParser rawResponsePayloadParser; + + public RawResponseRawDataConverter(SurveyUnitApiPort surveyUnitApiPort, + RawResponsePayloadParser rawResponsePayloadParser) { + super(surveyUnitApiPort); + this.rawResponsePayloadParser = rawResponsePayloadParser; + } + + /** + * Like convertRawResponseAndCollectEmptyModels but ignoring the empty raw responses + */ + public List convertRawResponse( + String collectionInstrumentId, + List rawResponseModels, + VariablesMap variablesMap + ) { + return convertRawResponseAndCollectEmptyModels( + collectionInstrumentId, + rawResponseModels, + variablesMap, + new ArrayList<>() + ); + } + + /** + * Converts RawResponseModels into SurveyUnitModels + * @param collectionInstrumentId Collection instrument id of raw responses to convert + * @param rawResponseModels raw responses to convert + * @param variablesMap variables map of the collection instrument + * @param emptySurveyUnitModels A list of survey units that will be filled with empty raw responses + * @return a list of SurveyUnitModels converted from raw responses + */ + public List convertRawResponseAndCollectEmptyModels( + String collectionInstrumentId, + List rawResponseModels, + VariablesMap variablesMap, + List emptySurveyUnitModels + ) { + List surveyUnitModels = new ArrayList<>(); + + Map> lastSurveyUnitModelsByInterrogationIdAndState = getLastSurveyUnitModels( + collectionInstrumentId, + rawResponseModels.stream().map(RawResponseModel::interrogationId).collect(Collectors.toList()) + ); + + for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) { + for (RawResponseModel rawResponseModel : rawResponseModels) { + SurveyUnitModel surveyUnitModel = buildSurveyUnitModel(rawResponseModel, dataState); + SurveyUnitModel lastSurveyUnitModelForDataState = null; + if(lastSurveyUnitModelsByInterrogationIdAndState.containsKey(rawResponseModel.interrogationId())){ + lastSurveyUnitModelForDataState = lastSurveyUnitModelsByInterrogationIdAndState + .get(rawResponseModel.interrogationId()) + .get(dataState); + } + + convertCollectedVariables( + rawResponseModel.payload(), + rawResponseModel.interrogationId(), + lastSurveyUnitModelForDataState, + surveyUnitModel, + dataState, + RawDataModelType.FILIERE, + variablesMap + ); + + if (dataState == DataState.COLLECTED) { + convertExternalVariables( + rawResponseModel.payload(), + lastSurveyUnitModelForDataState, + surveyUnitModel, + RawDataModelType.FILIERE, + variablesMap + ); + } + + boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() + && surveyUnitModel.getExternalVariables().isEmpty(); + + if (hasNoVariable) { + if (surveyUnitModel.getState() == DataState.COLLECTED) { + log.warn( + "No collected or external variable for interrogation {}, raw data is ignored.", + rawResponseModel.interrogationId() + ); + } + emptySurveyUnitModels.add(surveyUnitModel); + continue; + } + + surveyUnitModels.add(surveyUnitModel); + } + } + + return surveyUnitModels; + } + + private SurveyUnitModel buildSurveyUnitModel(RawResponseModel rawResponseModel, DataState dataState) { + String questionnaireStateString = + rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState"); + + RawResponseDto.QuestionnaireStateEnum questionnaireStateEnum = null; + try { + questionnaireStateEnum = RawResponseDto.QuestionnaireStateEnum.valueOf(questionnaireStateString); + } catch (IllegalArgumentException _) { + log.warn("'{}' is not a valid questionnaire state according to filiere model", questionnaireStateString); + } catch (NullPointerException _) { + //Nothing to do + } + + return SurveyUnitModel.builder() + .collectionInstrumentId(rawResponseModel.collectionInstrumentId()) + .majorModelVersion(rawResponsePayloadParser.getStringField(rawResponseModel, "majorModelVersion")) + .mode(rawResponseModel.mode()) + .interrogationId(rawResponseModel.interrogationId()) + .usualSurveyUnitId(rawResponsePayloadParser.getStringField(rawResponseModel, "usualSurveyUnitId")) + .questionnaireState(questionnaireStateEnum) + .validationDate(rawResponsePayloadParser.getValidationDate(rawResponseModel)) + .isCapturedIndirectly(rawResponsePayloadParser.getIsCapturedIndirectly(rawResponseModel)) + .state(dataState) + .fileDate(rawResponseModel.recordDate()) + .recordDate(Instant.now()) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + } +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java index a6e7d593b..80d92be6b 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java @@ -22,7 +22,6 @@ public interface LunaticJsonRawDataApiPort { void save(LunaticJsonRawDataModel rawData) throws GenesisException; List getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList); - List convertRawData(List rawData, VariablesMap variablesMap); List getLunaticJsonDataByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) throws NoDataException; List getUnprocessedDataIds(); Set getUnprocessedDataQuestionnaireIds(); @@ -34,7 +33,7 @@ public interface LunaticJsonRawDataApiPort { List getRawDataByInterrogationId(String interrogationId); - DataProcessResult processRawData(String collectionInstrumentId) throws GenesisException; + DataProcessResult processRawData(String questionnaireId) throws GenesisException; Map> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since); diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java index e3189b015..d5e1604d0 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java @@ -26,7 +26,11 @@ public interface SurveyUnitApiPort { List findByInterrogationId(String interrogationId); - List findLatestByIdAndByCollectionInstrumentId(String interrogationId, String collectionInstrumentId); + List findLatestByInterrogationIdAndCollectionInstrumentId(String interrogationId, String collectionInstrumentId); + List findLatestByInterrogationIds( + String collectionInstrumentOrQuestionnaireId, + Set interrogationIds + ); SurveyUnitSimplifiedDto findSimplified( String collectionInstrumentId, diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java index 0eed337b6..596b1cc35 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java @@ -17,12 +17,12 @@ public interface SurveyUnitPersistencePort { List findByUsualSurveyUnitAndCollectionInstrumentIds(String usualSurveyUnitId, String collectionInstrumentId); - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ - List findBySetOfIdsAndQuestionnaireIdAndMode(String questionnaireId, String mode, List interrogationIdSet); - //========= OPTIMISATIONS PERFS (START) ========== + List findByQuestionnaireIdAndModeAndInterrogationIds(String questionnaireId, String mode, List interrogationIdSet); + + List findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds(String collectionInstrumentOrQuestionnaireId, List interrogationIdSet); List findByInterrogationId(String interrogationId); diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index eb43894eb..ed6f4cbbe 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -169,6 +169,7 @@ private ProcessingResultDto processRawDataForMode( List emptySurveyUnitModels = new ArrayList<>(); List surveyUnitModels = lunaticJsonRawDataConverter.convertRawDataAndCollectEmptyModels( + questionnaireId, rawData, variablesMap, emptySurveyUnitModels @@ -218,14 +219,6 @@ private VariablesMap getVariablesMap( return variablesMap; } - @Override - public List convertRawData( - List rawDataList, - VariablesMap variablesMap - ) { - return lunaticJsonRawDataConverter.convertRawData(rawDataList, variablesMap); - } - @Override public void updateProcessDates(List surveyUnitModels) { Set collectionInstrumentIds = surveyUnitModels.stream() @@ -382,15 +375,4 @@ public Page findRawDataByCampaignIdAndDate( ) { return lunaticJsonRawDataPersistencePort.findByCampaignIdAndDate(campaignId, startDt, endDt, pageable); } - - public static String getValueString(Object value) { - if (value instanceof Double || value instanceof Float) { - BigDecimal bd = new BigDecimal(value.toString()); - return bd.stripTrailingZeros().toPlainString(); - } - if (value instanceof Number) { - return value.toString(); - } - return String.valueOf(value); - } } \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index 9450eb7fc..c1e6a2034 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -5,7 +5,7 @@ import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.dto.rawdata.ProcessingResultDto; import fr.insee.genesis.controller.utils.ControllerUtils; -import fr.insee.genesis.domain.converter.rawdata.RawResponseConverter; +import fr.insee.genesis.domain.converter.rawdata.RawResponseRawDataConverter; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; @@ -22,7 +22,6 @@ import fr.insee.genesis.exceptions.NoDataException; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.modelefiliere.ModeDto; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.domain.Page; @@ -49,13 +48,13 @@ public class RawResponseService implements RawResponseApiPort { private final SurveyUnitQualityToolService surveyUnitQualityToolService; private final FileUtils fileUtils; private final Config config; - private final RawResponseConverter rawResponseConverter; + private final RawResponseRawDataConverter rawResponseRawDataConverter; @Qualifier("rawResponseMongoAdapter") private final RawResponsePersistencePort rawResponsePersistencePort; //Lombok cannot use @Qualifier - public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, SurveyUnitQualityToolService surveyUnitQualityToolService, FileUtils fileUtils, Config config, RawResponseConverter rawResponseConverter, RawResponsePersistencePort rawResponsePersistencePort) { + public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, SurveyUnitQualityToolService surveyUnitQualityToolService, FileUtils fileUtils, Config config, RawResponseRawDataConverter rawResponseRawDataConverter, RawResponsePersistencePort rawResponsePersistencePort) { this.controllerUtils = controllerUtils; this.metadataService = metadataService; this.surveyUnitService = surveyUnitService; @@ -63,7 +62,7 @@ public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadata this.surveyUnitQualityToolService = surveyUnitQualityToolService; this.fileUtils = fileUtils; this.config = config; - this.rawResponseConverter = rawResponseConverter; + this.rawResponseRawDataConverter = rawResponseRawDataConverter; this.rawResponsePersistencePort = rawResponsePersistencePort; } @@ -169,7 +168,12 @@ private ProcessingResultDto processRawResponsesForMode( List emptySurveyUnitModels = new ArrayList<>(); List surveyUnitModels = - rawResponseConverter.convertRawResponseAndCollectEmptyModels(rawResponseModels, variablesMap, emptySurveyUnitModels); + rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + collectionInstrumentId, + rawResponseModels, + variablesMap, + emptySurveyUnitModels + ); surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); surveyUnitService.saveSurveyUnits(surveyUnitModels); diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index fca387fac..8fb8d66b7 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -88,9 +88,32 @@ public List findByInterrogationId(String interrogationId) { * @return the latest update for each variable of a survey unit */ @Override - public List findLatestByIdAndByCollectionInstrumentId(String interrogationId, String collectionInstrumentId) { - List latestUpdatesbyVariables = new ArrayList<>(); + public List findLatestByInterrogationIdAndCollectionInstrumentId(String interrogationId, String collectionInstrumentId) { List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, collectionInstrumentId); + return getLatestSurveyUnitModels(surveyUnitModels); + } + + /** + * @param collectionInstrumentOrQuestionnaireId : Collection instrument id/questionnaire id + * @param interrogationIdSet : Interrogation ids + * @return the latest updates for given interrogationIds + */ + @Override + public List findLatestByInterrogationIds( + String collectionInstrumentOrQuestionnaireId, + Set interrogationIdSet + ) { + List surveyUnitModels = surveyUnitPersistencePort.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + collectionInstrumentOrQuestionnaireId, + interrogationIdSet.stream().toList() + ); + + return getLatestSurveyUnitModels(surveyUnitModels); + } + + private List getLatestSurveyUnitModels(List surveyUnitModels) { + List latestUpdatesByVariables = new ArrayList<>(); + List modes = getDistinctsModes(surveyUnitModels); modes.forEach(mode ->{ List suByMode = surveyUnitModels.stream() @@ -99,7 +122,7 @@ public List findLatestByIdAndByCollectionInstrumentId(String in .toList(); //We had all the variables of the oldest update - latestUpdatesbyVariables.add(suByMode.getFirst()); + latestUpdatesByVariables.add(suByMode.getFirst()); //We keep the name of already added variables to skip them in older updates Set addedVariables = new HashSet<>(); SurveyUnitModel latestUpdate = suByMode.getFirst(); @@ -152,11 +175,11 @@ public List findLatestByIdAndByCollectionInstrumentId(String in if (!collectedVariablesToKeep.isEmpty() || !externalVariablesToKeep.isEmpty()){ surveyUnitModel.setCollectedVariables(collectedVariablesToKeep); surveyUnitModel.setExternalVariables(externalVariablesToKeep); - latestUpdatesbyVariables.add(surveyUnitModel); + latestUpdatesByVariables.add(surveyUnitModel); } }); }); - return latestUpdatesbyVariables; + return latestUpdatesByVariables; } private void addDataStateIntoList(List variableModelList, DataState state){ @@ -197,7 +220,7 @@ public SurveyUnitSimplifiedDto findSimplified( String interrogationId, Mode mode, Instant recordedBefore) throws NoDataException { - List responses = findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); + List responses = findLatestByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId); if(responses.isEmpty()){ String errorMessage = "No response found for interrogation %s".formatted(interrogationId); @@ -294,7 +317,7 @@ public List> findLatestByIdAndByQuestionnaireIdAndModeOrde List queryInParam = interrogationIds.stream().map(InterrogationId::getInterrogationId).toList(); //Get !!!all versions!!! of a set of "interrogationIds" - List allResponsesVersionsSet = surveyUnitPersistencePort.findBySetOfIdsAndQuestionnaireIdAndMode(questionnaireId, mode, queryInParam); + List allResponsesVersionsSet = surveyUnitPersistencePort.findByQuestionnaireIdAndModeAndInterrogationIds(questionnaireId, mode, queryInParam); //2) FILTER BY interrogationId AND ORDER BY DATE (MOST RECENT FIRST, oldest last) interrogationIds.forEach(interrogationId -> { diff --git a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java index e5ba1b6d2..e2667740f 100644 --- a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java +++ b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java @@ -1,5 +1,6 @@ package fr.insee.genesis.domain.utils; +import fr.insee.bpm.metadata.model.Variable; import fr.insee.bpm.metadata.model.VariableType; import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.Constants; @@ -45,7 +46,7 @@ public class DataVerifier { * @param variablesMap VariablesMap containing definitions of each variable */ public static void verifySurveyUnits(List surveyUnitModelsList, VariablesMap variablesMap){ - List surveyUnitModelsListFormatted = new ArrayList<>(); // Created FORCED SU models + List surveyUnitModelsListFormatted = new ArrayList<>(); // Created FORMATTED SU models for(String interrogationId : getInterrogationIds(surveyUnitModelsList)) { // For each id of the list List srcSurveyUnitModelsOfInterrogationId = surveyUnitModelsList.stream().filter(element -> element.getInterrogationId().equals(interrogationId)).toList(); @@ -56,7 +57,7 @@ public static void verifySurveyUnits(List surveyUnitModelsList, collectedVariablesManagement(srcSurveyUnitModelsOfInterrogationId, variablesMap, correctedCollectedVariables); externalVariablesManagement(srcSurveyUnitModelsOfInterrogationId, variablesMap, correctedExternalVariables); - //Create FORCED if any corrected variable + //Create FORMATTED if any corrected variable if(!correctedCollectedVariables.isEmpty() || !correctedExternalVariables.isEmpty()){ SurveyUnitModel newFormattedSurveyUnitModel = createFormattedSurveyUnitModel(surveyUnitModelsList, interrogationId, correctedCollectedVariables, correctedExternalVariables); surveyUnitModelsListFormatted.add(newFormattedSurveyUnitModel); @@ -153,8 +154,7 @@ private static void collectedVariablesManagement(List srcSurvey { VariableModel correctedCollectedVariable = verifyVariable( collectedVariableToVerify.variableModel(), - variablesMap.getVariable(collectedVariableToVerify.variableModel().varId()), - collectedVariableToVerify.dataState() + variablesMap.getVariable(collectedVariableToVerify.variableModel().varId()) ); if(correctedCollectedVariable != null){ @@ -191,10 +191,9 @@ private static void addIteration(VariableModel variableToCheck, private static VariableModel verifyVariable( VariableModel variableModel, - fr.insee.bpm.metadata.model.Variable variableDefinition, - DataState dataState + Variable variableDefinition ) { - if(isParseError(variableModel.value(), variableDefinition.getType(),dataState)){ + if(isParseError(variableModel.value(), variableDefinition.getType())){ return VariableModel.builder() .varId(variableModel.varId()) .value("") @@ -219,8 +218,7 @@ private static void externalVariablesManagement(List srcSuModel if(variablesMap.hasVariable(externalVariable.varId())) { VariableModel correctedExternalVariable = verifyVariable( externalVariable, - variablesMap.getVariable(externalVariable.varId()), - state + variablesMap.getVariable(externalVariable.varId()) ); if (correctedExternalVariable != null) { correctedExternalVariables.add(correctedExternalVariable); @@ -234,15 +232,12 @@ private static void externalVariablesManagement(List srcSuModel * Use the correct parser and try to parse * @param value value to verify * @param type type of the variable - * @param state state of the data where the variable is contained in * @return true if the value is not conform to the variable type */ - private static boolean isParseError(String value, VariableType type, DataState state){ + private static boolean isParseError(String value, VariableType type){ //Allow null values if(value == null){ - return !(state.equals(DataState.EDITED) - || state.equals(DataState.FORCED) - || state.equals(DataState.FORMATTED)); //Return false if datastate one of those + return false; } switch(type){ case BOOLEAN: diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index f74b8d18a..d2becf426 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -73,12 +73,24 @@ public List findByUsualSurveyUnitAndCollectionInstrumentIds(Str * @author Adrien Marchal */ @Override - public List findBySetOfIdsAndQuestionnaireIdAndMode(String questionnaireId, String mode, List interrogationIdSet) { + public List findByQuestionnaireIdAndModeAndInterrogationIds(String questionnaireId, String mode, List interrogationIdSet) { List surveyUnits = mongoRepository.findBySetOfIdsAndQuestionnaireIdAndMode(questionnaireId, mode, interrogationIdSet); return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); } //========= OPTIMISATIONS PERFS (END) ========== + @Override + public List findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + String collectionInstrumentOrQuestionnaireId, List interrogationIds + ) { + List surveyUnits = new ArrayList<>(); + + surveyUnits.addAll(mongoRepository.findByQuestionnaireIdAndInterrogationIds(collectionInstrumentOrQuestionnaireId, interrogationIds)); + surveyUnits.addAll(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds(collectionInstrumentOrQuestionnaireId, interrogationIds)); + + return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); + } + @Override public List findByInterrogationId(String interrogationId) { List surveyUnits = mongoRepository.findByInterrogationId(interrogationId); diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java index e2e200426..47218777b 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java @@ -30,9 +30,25 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findBySetOfIdsAndQuestionnaireIdAndMode(String questionnaireId, String mode, List interrogationIdSet); + List findBySetOfIdsAndQuestionnaireIdAndMode( + String questionnaireId, + String mode, + List interrogationIds + ); //========= OPTIMISATIONS PERFS (END) ========== + @Query("{ 'questionnaireId' : ?0, 'interrogationId' : { $in: ?2 } }") + List findByQuestionnaireIdAndInterrogationIds( + String questionnaireId, + List interrogationIds + ); + @Query("{ 'collectionInstrumentId' : ?0, 'interrogationId' : { $in: ?2 } }") + List findByCollectionInstrumentIdAndInterrogationIds( + String collectionInstrumentId, + List interrogationIds + ); + + @Query(value = "{ 'questionnaireId' : ?0 }", fields = "{ 'interrogationId' : 1, 'mode' : 1 }") List findInterrogationIdsByQuestionnaireId(String questionnaireId); diff --git a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java index aeb6345f8..897507762 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java +++ b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java @@ -6,6 +6,7 @@ import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; @@ -54,6 +55,7 @@ class ControllerAccessIT extends IntegrationTestAbstract { private RawResponseInputRepository rawRepository; @MockitoBean private SurveyUnitQualityToolService surveyUnitQualityToolService; + /** * Provides a stream of URIs that are allowed for reader. */ diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java index db58fdfb3..6b32284a5 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java @@ -231,7 +231,7 @@ void saveLunaticJson_syntax_or_json_schema_error_test(String jsonFilePathString) @Nested @DisplayName("Filiere model raw data processing tests") - class rawDataProcessingTests { + class RawResponsesProcessingTests { //HAPPY PATHS @Test @WithMockUser(roles = "SCHEDULER") @@ -258,7 +258,8 @@ void process_raw_response_interrogation_list_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -329,7 +330,8 @@ void process_raw_response_interrogation_list_no_collected_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -397,7 +399,8 @@ void process_raw_response_collectionInstrumentId_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -443,6 +446,147 @@ void process_raw_response_collectionInstrumentId_test(){ } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @WithMockUser(roles = "SCHEDULER") + @DisplayName("Filiere model raw data null values should be kept if null or non null value already exists") + @SneakyThrows + void process_raw_response_keep_null_test(boolean isNullSurveyUnitValues) { + //GIVEN + String collectionInstrumentId = "TESTQUEST"; + Mode mode = Mode.WEB; + List interrogationIds = List.of("INTERRO1"); + + //Raw responses with null variables + String variableName = "VAR1"; + String collectedValue = "value1"; + Map collectedVariablesAndValues = new HashMap<>(); + collectedVariablesAndValues.put(variableName, collectedValue); + + String externalVariableName = "EXTVAR1"; + String externalValue = "externalvalue1"; + Map externalVariablesAndValues = new HashMap<>(); + externalVariablesAndValues.put(externalVariableName, externalValue); + + setFiliereModelTestMockBehaviour( + collectionInstrumentId, + mode, + interrogationIds, + collectedVariablesAndValues, + externalVariablesAndValues, + true + ); + + //Survey unit that already exists for first interrogation + SurveyUnitDocument alreadyPresentSurveyUnitDocument = getSurveyUnitDocument( + collectionInstrumentId, + interrogationIds.getFirst(), + variableName, + isNullSurveyUnitValues ? null : collectedValue, + externalVariableName, + isNullSurveyUnitValues ? null : externalValue + ); + + when(surveyUnitMongoDBRepository.findByCollectionInstrumentIdAndInterrogationIds(collectionInstrumentId, interrogationIds)) + .thenReturn(List.of(alreadyPresentSurveyUnitDocument)); + + // WHEN + mockMvc.perform(post("/raw-responses/%s/process".formatted(collectionInstrumentId)) + .with(csrf()) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + + //THEN + @SuppressWarnings("unchecked") + ArgumentCaptor> listArgumentCaptor = + ArgumentCaptor.forClass(List.class); + verify(surveyUnitMongoDBRepository, times(1)) + .insert(listArgumentCaptor.capture()); + + //Document must have null variables values + List savedDocuments = listArgumentCaptor.getValue(); + Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); + SurveyUnitDocument savedDocument = savedDocuments.stream().filter( + surveyUnitDocument -> + surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst())) + .toList().getFirst(); + Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); + VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); + Assertions.assertThat(variableDocument.getValue()).isNull(); + Assertions.assertThat(savedDocument.getExternalVariables()).isNotNull().hasSize(1); + variableDocument = savedDocument.getExternalVariables().getFirst(); + Assertions.assertThat(variableDocument.getValue()).isNull(); + } + + @Test + @WithMockUser(roles = "SCHEDULER") + @DisplayName("Filiere model raw data shouldn't overwrite with null if not existent before") + @SneakyThrows + void process_raw_response_null_on_absent_test() { + //GIVEN + String collectionInstrumentId = "TESTQUEST"; + Mode mode = Mode.WEB; + List interrogationIds = List.of("INTERRO1"); + + //Raw responses with null variables + String variableName = "VAR1"; + Map collectedVariablesAndValues = new HashMap<>(); + collectedVariablesAndValues.put(variableName, null); + + String externalVariableName = "EXTVAR1"; + Map externalVariablesAndValues = new HashMap<>(); + externalVariablesAndValues.put(externalVariableName, null); + + setFiliereModelTestMockBehaviour( + collectionInstrumentId, + mode, + interrogationIds, + collectedVariablesAndValues, + externalVariablesAndValues, + true + ); + + //Survey unit that already exists for first interrogation with variables not present in raw + SurveyUnitDocument alreadyPresentSurveyUnitDocument = getSurveyUnitDocument( + collectionInstrumentId, + interrogationIds.getFirst(), + "VAR2", + "value", + "EXTVAR2", + "externalValue" + ); + + when(surveyUnitMongoDBRepository.findByCollectionInstrumentIdAndInterrogationIds(collectionInstrumentId, interrogationIds)) + .thenReturn(List.of(alreadyPresentSurveyUnitDocument)); + + // WHEN + mockMvc.perform(post("/raw-responses/%s/process".formatted(collectionInstrumentId)) + .with(csrf()) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + + //THEN + @SuppressWarnings("unchecked") + ArgumentCaptor> listArgumentCaptor = + ArgumentCaptor.forClass(List.class); + verify(surveyUnitMongoDBRepository, times(1)) + .insert(listArgumentCaptor.capture()); + + //Document shouldn't have null variables + List savedDocuments = listArgumentCaptor.getValue(); + Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); + SurveyUnitDocument savedDocument = savedDocuments.stream().filter( + surveyUnitDocument -> + surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst())) + .toList().getFirst(); + Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); + VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); + Assertions.assertThat(variableDocument.getVarId()).isNotEqualTo(variableName); + Assertions.assertThat(savedDocument.getExternalVariables()).isNotNull().hasSize(1); + variableDocument = savedDocument.getExternalVariables().getFirst(); + Assertions.assertThat(variableDocument.getVarId()).isNotEqualTo(externalVariableName); + } + @Test @WithMockUser(roles = "SCHEDULER") @DisplayName("Filiere model raw data should have processed date if no data") @@ -463,7 +607,8 @@ void process_raw_response_noData_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -496,7 +641,8 @@ private void setFiliereModelTestMockBehaviour( Mode mode, List interrogationIds, Map collectedVariableNamesAndValues, - Map externalVariableNamesAndValues + Map externalVariableNamesAndValues, + boolean isNullValues ){ //Disable withReview DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument(); @@ -545,6 +691,29 @@ private void setFiliereModelTestMockBehaviour( ).thenReturn(interrogationIds); //FILIERE RAW DOCS + List rawResponseDocuments = getRawResponseDocuments( + collectionInstrumentId, + mode, + interrogationIds, + collectedVariableNamesAndValues, + externalVariableNamesAndValues, + isNullValues + ); + when(rawResponseRepository.findByCollectionInstrumentIdAndModeAndInterrogationIdList( + eq(collectionInstrumentId), + eq(mode.getJsonName()), + argThat(argument -> argument.containsAll(interrogationIds)) //Any order + )).thenReturn(rawResponseDocuments); + } + + private List getRawResponseDocuments( + String collectionInstrumentId, + Mode mode, + List interrogationIds, + Map collectedVariableNamesAndValues, + Map externalVariableNamesAndValues, + boolean isNullValues + ) { List rawResponseDocuments = new ArrayList<>(); for(String interrogationId : interrogationIds){ Map dataMap = getNewDataMap(); @@ -552,14 +721,14 @@ private void setFiliereModelTestMockBehaviour( for(Map.Entry variable : collectedVariableNamesAndValues.entrySet()) { String variableName = variable.getKey(); //To have different value on each interrogation - String value = interrogationId + variable.getValue(); + String value = isNullValues ? null : interrogationId + variable.getValue(); addVariableToDataMap(dataMap, variableName, value, Constants.COLLECTED_NODE_NAME); } //EXTERNAL VARIABLES for(Map.Entry variable : externalVariableNamesAndValues.entrySet()) { String variableName = variable.getKey(); - String value = interrogationId + variable.getValue(); + String value = isNullValues ? null : interrogationId + variable.getValue(); addVariableToDataMap(dataMap, variableName, value, Constants.EXTERNAL_NODE_NAME); } @@ -581,17 +750,13 @@ private void setFiliereModelTestMockBehaviour( .build(); rawResponseDocuments.add(rawResponseDocument); } - when(rawResponseRepository.findByCollectionInstrumentIdAndModeAndInterrogationIdList( - eq(collectionInstrumentId), - eq(mode.getJsonName()), - argThat(argument -> argument.containsAll(interrogationIds)) //Any order - )).thenReturn(rawResponseDocuments); + return rawResponseDocuments; } } @Nested @DisplayName("Old model raw data processing tests") - class lunaticJsonDataProcessingTests { + class LunaticJsonDataProcessingTests { @Test @WithMockUser(roles = "SCHEDULER") @DisplayName("Old model raw data should be processed using the questionnaireId") @@ -618,7 +783,8 @@ void processLunaticJsonRawData_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -664,6 +830,147 @@ void processLunaticJsonRawData_test(){ } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @WithMockUser(roles = "SCHEDULER") + @DisplayName("Old model raw data null values should be kept if non null value already exists") + @SneakyThrows + void processLunaticJsonRawData_keep_null_test(boolean isNullSurveyUnitValues){ + //GIVEN + String questionnaireId = "TESTQUEST"; + Mode mode = Mode.WEB; + + List interrogationIds = List.of("INTERRO1"); + + String variableName = "VAR1"; + String collectedValue = "value1"; + Map collectedVariablesAndValues = new HashMap<>(); + collectedVariablesAndValues.put(variableName, collectedValue); + + String externalVariableName = "EXTVAR1"; + String externalValue = "externalvalue1"; + Map externalVariablesAndValues = new HashMap<>(); + externalVariablesAndValues.put(externalVariableName, externalValue); + + setOldModelTestMockBehaviour( + questionnaireId, + mode, + interrogationIds, + collectedVariablesAndValues, + externalVariablesAndValues, + true + ); + + //Survey unit that already exists for first interrogation + SurveyUnitDocument alreadyPresentSurveyUnitDocument = getSurveyUnitDocument( + questionnaireId, + interrogationIds.getFirst(), + variableName, + isNullSurveyUnitValues ? null : collectedValue, + externalVariableName, + isNullSurveyUnitValues ? null : externalValue + ); + when(surveyUnitMongoDBRepository.findByCollectionInstrumentIdAndInterrogationIds(questionnaireId, interrogationIds)) + .thenReturn(List.of(alreadyPresentSurveyUnitDocument)); + + // WHEN + mockMvc.perform(post("/responses/raw/lunatic-json/%s/process".formatted(questionnaireId)) + .with(csrf()) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + + //THEN + @SuppressWarnings("unchecked") + ArgumentCaptor> listArgumentCaptor = + ArgumentCaptor.forClass(List.class); + verify(surveyUnitMongoDBRepository, times(1)) + .insert(listArgumentCaptor.capture()); + + //Document must have null variables values + List savedDocuments = listArgumentCaptor.getValue(); + Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); + SurveyUnitDocument savedDocument = savedDocuments.stream().filter( + surveyUnitDocument -> + surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst())) + .toList().getFirst(); + Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); + VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); + Assertions.assertThat(variableDocument.getValue()).isNull(); + Assertions.assertThat(savedDocument.getExternalVariables()).isNotNull().hasSize(1); + variableDocument = savedDocument.getExternalVariables().getFirst(); + Assertions.assertThat(variableDocument.getValue()).isNull(); + } + + @Test + @WithMockUser(roles = "SCHEDULER") + @DisplayName("Old model raw data null values should be kept if non null value already exists") + @SneakyThrows + void processLunaticJsonRawData_null_on_absent_test(){ + //GIVEN + String questionnaireId = "TESTQUEST"; + Mode mode = Mode.WEB; + + List interrogationIds = List.of("INTERRO1"); + + String variableName = "VAR1"; + String collectedValue = "value1"; + Map collectedVariablesAndValues = new HashMap<>(); + collectedVariablesAndValues.put(variableName, collectedValue); + + String externalVariableName = "EXTVAR1"; + String externalValue = "externalvalue1"; + Map externalVariablesAndValues = new HashMap<>(); + externalVariablesAndValues.put(externalVariableName, externalValue); + + setOldModelTestMockBehaviour( + questionnaireId, + mode, + interrogationIds, + collectedVariablesAndValues, + externalVariablesAndValues, + true + ); + + //Survey unit that already exists for first interrogation + SurveyUnitDocument alreadyPresentSurveyUnitDocument = getSurveyUnitDocument( + questionnaireId, + interrogationIds.getFirst(), + "VAR2", + "value", + "EXTVAR2", + "externalValue" + ); + when(surveyUnitMongoDBRepository.findByCollectionInstrumentIdAndInterrogationIds(questionnaireId, interrogationIds)) + .thenReturn(List.of(alreadyPresentSurveyUnitDocument)); + + // WHEN + mockMvc.perform(post("/responses/raw/lunatic-json/%s/process".formatted(questionnaireId)) + .with(csrf()) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + + //THEN + @SuppressWarnings("unchecked") + ArgumentCaptor> listArgumentCaptor = + ArgumentCaptor.forClass(List.class); + verify(surveyUnitMongoDBRepository, times(1)) + .insert(listArgumentCaptor.capture()); + + //Document shouldn't have null variables + List savedDocuments = listArgumentCaptor.getValue(); + Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size()); + SurveyUnitDocument savedDocument = savedDocuments.stream().filter( + surveyUnitDocument -> + surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst())) + .toList().getFirst(); + Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1); + VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst(); + Assertions.assertThat(variableDocument.getVarId()).isNotEqualTo(variableName); + Assertions.assertThat(savedDocument.getExternalVariables()).isNotNull().hasSize(1); + variableDocument = savedDocument.getExternalVariables().getFirst(); + Assertions.assertThat(variableDocument.getVarId()).isNotEqualTo(externalVariableName); + } + @Test @WithMockUser(roles = "SCHEDULER") @DisplayName("Old model raw data should be processed even without collected") @@ -688,7 +995,8 @@ void processLunaticJsonRawData_no_collected_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -754,7 +1062,8 @@ void processLunaticJsonRawData_noData_test(){ mode, interrogationIds, collectedVariablesAndValues, - externalVariablesAndValues + externalVariablesAndValues, + false ); // WHEN @@ -791,7 +1100,8 @@ private void setOldModelTestMockBehaviour(String questionnaireId, Mode mode, List interrogationIds, Map collectedVariableNamesAndValues, - Map externalVariableNamesAndValues + Map externalVariableNamesAndValues, + boolean isNullValues ){ //Disable withReview DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument(); @@ -845,6 +1155,29 @@ private void setOldModelTestMockBehaviour(String questionnaireId, .thenReturn(List.of(groupedInterrogationDocument)); //LUNATIC JSON RAW DOCS + List lunaticJsonRawDataDocuments = getLunaticJsonRawDataDocuments( + questionnaireId, + mode, + interrogationIds, + collectedVariableNamesAndValues, + externalVariableNamesAndValues, + isNullValues + ); + when(lunaticJsonMongoDBRepository.findByQuestionnaireModeAndInterrogations( + eq(questionnaireId), + eq(mode), + anyList() + )).thenReturn(lunaticJsonRawDataDocuments); + } + + private List getLunaticJsonRawDataDocuments( + String questionnaireId, + Mode mode, + List interrogationIds, + Map collectedVariableNamesAndValues, + Map externalVariableNamesAndValues, + boolean isNullValues + ) { List lunaticJsonRawDataDocuments = new ArrayList<>(); for(String interrogationId : interrogationIds){ Map dataMap = getNewDataMap(); @@ -852,14 +1185,14 @@ private void setOldModelTestMockBehaviour(String questionnaireId, for(Map.Entry variable : collectedVariableNamesAndValues.entrySet()) { String variableName = variable.getKey(); //To have different value on each interrogation - String value = interrogationId + variable.getValue(); + String value = isNullValues ? null : interrogationId + variable.getValue(); addVariableToDataMap(dataMap, variableName, value, Constants.COLLECTED_NODE_NAME); } //EXTERNAL VARIABLES for(Map.Entry variable : externalVariableNamesAndValues.entrySet()) { String variableName = variable.getKey(); - String value = interrogationId + variable.getValue(); + String value = isNullValues ? null : interrogationId + variable.getValue(); addVariableToDataMap(dataMap, variableName, value, Constants.EXTERNAL_NODE_NAME); } @@ -875,11 +1208,7 @@ private void setOldModelTestMockBehaviour(String questionnaireId, .build(); lunaticJsonRawDataDocuments.add(lunaticJsonRawDataDocument); } - when(lunaticJsonMongoDBRepository.findByQuestionnaireModeAndInterrogations( - eq(questionnaireId), - eq(mode), - anyList() - )).thenReturn(lunaticJsonRawDataDocuments); + return lunaticJsonRawDataDocuments; } } @@ -928,5 +1257,37 @@ private void addExternalVariableToDataMap(Map dataMap, externalMap.put(variableName, value); } + private SurveyUnitDocument getSurveyUnitDocument( + String collectionInstrumentId, + String interrogationId, + String variableName, + String collectedValue, + String externalVariableName, + String externalValue + ) { + SurveyUnitDocument alreadyPresentSurveyUnitDocument = new SurveyUnitDocument(); + alreadyPresentSurveyUnitDocument.setCollectionInstrumentId(collectionInstrumentId); + alreadyPresentSurveyUnitDocument.setInterrogationId(interrogationId); + alreadyPresentSurveyUnitDocument.setMode(Mode.WEB.getModeName()); + alreadyPresentSurveyUnitDocument.setState("COLLECTED"); + + alreadyPresentSurveyUnitDocument.setCollectedVariables(new ArrayList<>()); + VariableDocument oldVariable = new VariableDocument(); + oldVariable.setVarId(variableName); + oldVariable.setIteration(1); + oldVariable.setValue(collectedValue); + oldVariable.setScope(Constants.ROOT_GROUP_NAME); + alreadyPresentSurveyUnitDocument.getCollectedVariables().add(oldVariable); + + alreadyPresentSurveyUnitDocument.setExternalVariables(new ArrayList<>()); + oldVariable = new VariableDocument(); + oldVariable.setVarId(externalVariableName); + oldVariable.setIteration(1); + oldVariable.setValue(externalValue); + oldVariable.setScope(Constants.ROOT_GROUP_NAME); + alreadyPresentSurveyUnitDocument.getExternalVariables().add(oldVariable); + return alreadyPresentSurveyUnitDocument; + } + //TODO GET tests } \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerIT.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerIT.java index ac1b75ec2..7e9978a72 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerIT.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerIT.java @@ -117,6 +117,81 @@ void get_simplified_response_test() { } + @Test + @WithMockUser(roles = "USER_KRAFTWERK") + @DisplayName("Get simplified response with null variable") + @SneakyThrows + void get_simplified_response_null_variable_test() { + //GIVEN + String collectionInstrumentId = "collectionInstrumentId"; + String interrogationId = "interrogationId"; + String usualSurveyUnitId = "usualSurveyUnitId"; + Mode mode = Mode.WEB; + RawResponseDto.QuestionnaireStateEnum questionnaireStateEnum = + RawResponseDto.QuestionnaireStateEnum.FINISHED; + LocalDateTime validationDate = LocalDateTime.now().minusDays(2); + DataState dataState = DataState.COLLECTED; + + SurveyUnitDocument surveyUnitDocument = new SurveyUnitDocument(); + surveyUnitDocument.setCollectionInstrumentId(collectionInstrumentId); + surveyUnitDocument.setInterrogationId(interrogationId); + surveyUnitDocument.setMode(mode.getModeName()); + surveyUnitDocument.setState(dataState.name()); + surveyUnitDocument.setUsualSurveyUnitId(usualSurveyUnitId); + surveyUnitDocument.setQuestionnaireState(questionnaireStateEnum); + surveyUnitDocument.setValidationDate(validationDate); + surveyUnitDocument.setCollectedVariables(new ArrayList<>()); + surveyUnitDocument.setExternalVariables(new ArrayList<>()); + surveyUnitDocument.setRecordDate(Instant.now().minusSeconds(60)); + + String collectedVariableName = "var1"; + VariableDocument collectedVariableDocument = new VariableDocument(); + collectedVariableDocument.setVarId(collectedVariableName); + collectedVariableDocument.setValue(null); //null value + collectedVariableDocument.setIteration(1); + collectedVariableDocument.setScope(Constants.ROOT_GROUP_NAME); + surveyUnitDocument.getCollectedVariables().add(collectedVariableDocument); + + String externalVariableName = "extvar1"; + VariableDocument externalVariableDocument = new VariableDocument(); + externalVariableDocument.setVarId(externalVariableName); + externalVariableDocument.setValue(null); //null value + externalVariableDocument.setIteration(1); + externalVariableDocument.setScope(Constants.ROOT_GROUP_NAME); + surveyUnitDocument.getExternalVariables().add(externalVariableDocument); + + + Mockito.when(surveyUnitMongoDBRepository.findByInterrogationIdAndCollectionInstrumentId( + interrogationId, collectionInstrumentId + )).thenReturn(List.of(surveyUnitDocument)); + + //WHEN + THEN + mockMvc.perform(get("/responses/%s/%s/%s" + .formatted(collectionInstrumentId, mode.getModeName(), interrogationId) + ) + .with(csrf())) + .andExpect(status().isOk()) + + .andExpect(jsonPath("$.collectionInstrumentId").value(collectionInstrumentId)) + .andExpect(jsonPath("$.interrogationId").value(interrogationId)) + .andExpect(jsonPath("$.usualSurveyUnitId").value(usualSurveyUnitId)) + .andExpect(jsonPath("$.mode").value(mode.getModeName())) + .andExpect(jsonPath("$.validationDate").value(validationDate.format(DateTimeFormatter.ISO_DATE_TIME))) + .andExpect(jsonPath("$.questionnaireState").value(questionnaireStateEnum.name())) + + .andExpect(jsonPath("$.variablesUpdate[0].varId").value(collectedVariableName)) + .andExpect(jsonPath("$.variablesUpdate[0].value").isEmpty()) + .andExpect(jsonPath("$.variablesUpdate[0].state").value(dataState.name())) + .andExpect(jsonPath("$.variablesUpdate[0].scope").value(Constants.ROOT_GROUP_NAME)) + .andExpect(jsonPath("$.variablesUpdate[0].iteration").value(1)) + + .andExpect(jsonPath("$.externalVariables[0].varId").value(externalVariableName)) + .andExpect(jsonPath("$.externalVariables[0].value").isEmpty()) + .andExpect(jsonPath("$.externalVariables[0].state").value(dataState.name())) + .andExpect(jsonPath("$.externalVariables[0].scope").value(Constants.ROOT_GROUP_NAME)) + .andExpect(jsonPath("$.externalVariables[0].iteration").value(1)); + } + @Test @WithMockUser(roles = "USER_KRAFTWERK") @DisplayName("Get simplified response with an additionnal EDITED document") diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index c57fd2d5a..63aa80e35 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -278,7 +278,7 @@ class GetLatestByInterrogationAndCollectionInstrumentTests { @DisplayName("Should return 200 with list of models") void getLatest_shouldReturn200() throws Exception { // GIVEN - when(surveyUnitApiPort.findLatestByIdAndByCollectionInstrumentId("INTERRO01", "QUEST01")) + when(surveyUnitApiPort.findLatestByInterrogationIdAndCollectionInstrumentId("INTERRO01", "QUEST01")) .thenReturn(List.of()); // WHEN / THEN @@ -298,7 +298,7 @@ class GetLatestByInterrogationOneObjectTests { void getLatestOneObject_shouldReturn200AndAggregateVariables() throws Exception { // GIVEN SurveyUnitModel model = buildSurveyUnitModel("INTERRO01", "QUEST01", Mode.WEB, DataState.COLLECTED); - when(surveyUnitApiPort.findLatestByIdAndByCollectionInstrumentId("INTERRO01", "QUEST01")) + when(surveyUnitApiPort.findLatestByInterrogationIdAndCollectionInstrumentId("INTERRO01", "QUEST01")) .thenReturn(List.of(model)); // WHEN / THEN @@ -316,7 +316,7 @@ void getLatestOneObject_shouldFilterByMode() throws Exception { // GIVEN SurveyUnitModel webModel = buildSurveyUnitModel("INTERRO01", "QUEST01", Mode.WEB, DataState.COLLECTED); SurveyUnitModel paperModel = buildSurveyUnitModel("INTERRO01", "QUEST01", Mode.PAPER, DataState.COLLECTED); - when(surveyUnitApiPort.findLatestByIdAndByCollectionInstrumentId("INTERRO01", "QUEST01")) + when(surveyUnitApiPort.findLatestByInterrogationIdAndCollectionInstrumentId("INTERRO01", "QUEST01")) .thenReturn(List.of(webModel, paperModel)); // WHEN / THEN @@ -374,7 +374,7 @@ void getLatestList_shouldReturn200() throws Exception { SurveyUnitModel model = buildSurveyUnitModel("INTERRO01", "QUEST01", Mode.WEB, DataState.COLLECTED); when(surveyUnitApiPort.findModesByCollectionInstrumentId("QUEST01")) .thenReturn(List.of(Mode.WEB)); - when(surveyUnitApiPort.findLatestByIdAndByCollectionInstrumentId("INTERRO01", "QUEST01")) + when(surveyUnitApiPort.findLatestByInterrogationIdAndCollectionInstrumentId("INTERRO01", "QUEST01")) .thenReturn(List.of(model)); // WHEN / THEN @@ -399,7 +399,7 @@ void getLatestList_noVariables_shouldReturnEmptyList() throws Exception { .build(); when(surveyUnitApiPort.findModesByCollectionInstrumentId("QUEST01")) .thenReturn(List.of(Mode.WEB)); - when(surveyUnitApiPort.findLatestByIdAndByCollectionInstrumentId("INTERRO01", "QUEST01")) + when(surveyUnitApiPort.findLatestByInterrogationIdAndCollectionInstrumentId("INTERRO01", "QUEST01")) .thenReturn(List.of(modelNoVars)); // WHEN / THEN diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java index 2b761c481..3230ad697 100644 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java @@ -8,38 +8,56 @@ import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.parser.rawdata.LunaticJsonRawDataPayloadParser; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class LunaticJsonRawDataConverterTest { - public static final LocalDateTime DATE_TIME = LocalDateTime.parse("2025-01-01T10:00:00"); + private static final LocalDateTime DATE_TIME = LocalDateTime.parse("2025-01-01T10:00:00"); + private static final String QUESTIONNAIRE_ID = "testQuestionnaire"; + private static final String INTERROGATION_ID = "testInterrogation"; + + @Mock + private SurveyUnitService surveyUnitService; + @Mock private LunaticJsonRawDataPayloadParser payloadParser; + @InjectMocks + private LunaticJsonRawDataConverter converter; + @Test void shouldReturnNoSurveyUnitWhenRawDataListIsEmpty() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class); List emptySurveyUnitModels = new ArrayList<>(); List result = converter.convertRawDataAndCollectEmptyModels( + "test", List.of(), variablesMap, emptySurveyUnitModels @@ -51,7 +69,6 @@ void shouldReturnNoSurveyUnitWhenRawDataListIsEmpty() { @Test void shouldCreateCollectedAndEditedSurveyUnitsFromLegacyRawData() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class); LunaticJsonRawDataModel rawData = rawData(Map.of( @@ -60,45 +77,36 @@ void shouldCreateCollectedAndEditedSurveyUnitsFromLegacyRawData() { ) )); - try (MockedStatic rawResponseConverter = mockStatic(RawResponseConverter.class)) { - rawResponseConverter - .when(() -> RawResponseConverter.processCollectedVariable(any(), any(), any(), any(), any())) - .thenAnswer(invocation -> { - String state = invocation.getArgument(1); - List destination = invocation.getArgument(4); - - destination.add(VariableModel.builder() - .varId("FIRST_NAME") - .value(state) - .build()); - - return null; - }); - - List result = converter.convertRawData( - List.of(rawData), - variablesMap - ); + List result = converter.convertRawData( + QUESTIONNAIRE_ID, + List.of(rawData), + variablesMap + ); - assertThat(result).hasSize(2); + assertThat(result).hasSize(2); - assertThat(result) - .extracting(SurveyUnitModel::getState) - .containsExactly(DataState.COLLECTED, DataState.EDITED); + assertThat(result) + .extracting(SurveyUnitModel::getState) + .containsExactly(DataState.COLLECTED, DataState.EDITED); - assertThat(result.get(0).getCollectedVariables()) + for(DataState dataState : Set.of(DataState.COLLECTED, DataState.EDITED)){ + Optional surveyUnitModelOptional = result.stream().filter( + surveyUnitModel -> surveyUnitModel.getState().equals(dataState) + ).findFirst(); + + Assertions.assertThat(surveyUnitModelOptional).isPresent(); + SurveyUnitModel surveyUnitModel = surveyUnitModelOptional.get(); + Assertions.assertThat(surveyUnitModel.getCollectedVariables()) + .extracting(VariableModel::varId) + .containsExactly("FIRST_NAME"); + Assertions.assertThat(surveyUnitModel.getCollectedVariables()) .extracting(VariableModel::value) - .containsExactly("COLLECTED"); - - assertThat(result.get(1).getCollectedVariables()) - .extracting( VariableModel::value) - .containsExactly("EDITED"); + .containsExactly(dataState.equals(DataState.COLLECTED) ? "Alice" : "Alicia"); } } @Test void shouldConvertExternalVariablesOnlyForCollectedSurveyUnit() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); when(variablesMap.getVariable("COUNTRY").getGroupName()).thenReturn("ROOT"); @@ -115,6 +123,7 @@ void shouldConvertExternalVariablesOnlyForCollectedSurveyUnit() { List emptySurveyUnitModels = new ArrayList<>(); List result = converter.convertRawDataAndCollectEmptyModels( + QUESTIONNAIRE_ID, List.of(rawData), variablesMap, emptySurveyUnitModels @@ -144,7 +153,6 @@ void shouldConvertExternalVariablesOnlyForCollectedSurveyUnit() { @Test void shouldCollectEmptySurveyUnitsWhenNoCollectedNorExternalVariablesExist() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class); LunaticJsonRawDataModel rawData = rawData(Map.of( @@ -155,6 +163,7 @@ void shouldCollectEmptySurveyUnitsWhenNoCollectedNorExternalVariablesExist() { List emptySurveyUnitModels = new ArrayList<>(); List result = converter.convertRawDataAndCollectEmptyModels( + QUESTIONNAIRE_ID, List.of(rawData), variablesMap, emptySurveyUnitModels @@ -170,7 +179,6 @@ void shouldCollectEmptySurveyUnitsWhenNoCollectedNorExternalVariablesExist() { @Test void shouldReadFiliereRawDataFromNestedDataProperty() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); when(variablesMap.getVariable("COUNTRY").getGroupName()).thenReturn("ROOT"); @@ -185,6 +193,7 @@ void shouldReadFiliereRawDataFromNestedDataProperty() { List emptySurveyUnitModels = new ArrayList<>(); List result = converter.convertRawDataAndCollectEmptyModels( + QUESTIONNAIRE_ID, List.of(rawData), variablesMap, emptySurveyUnitModels @@ -199,7 +208,6 @@ void shouldReadFiliereRawDataFromNestedDataProperty() { @Test void shouldAssignRootScopeWhenExternalVariableIsMissingFromMetadata() { - LunaticJsonRawDataConverter converter = new LunaticJsonRawDataConverter(payloadParser); VariablesMap variablesMap = mock(VariablesMap.class); when(variablesMap.getVariable("UNKNOWN_VARIABLE")).thenReturn(null); @@ -210,6 +218,7 @@ void shouldAssignRootScopeWhenExternalVariableIsMissingFromMetadata() { )); List result = converter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), variablesMap ); @@ -222,15 +231,461 @@ void shouldAssignRootScopeWhenExternalVariableIsMissingFromMetadata() { .isEqualTo(Constants.ROOT_GROUP_NAME); } + @Nested + @DisplayName("Null cases tests") + class NullVariablesTests { + + //NullVariablesTests constants + private static final String COLLECTED_VARIABLE_NAME = "VAR1"; + private static final String COLLECTED_VARIABLE_VALUE = "test"; + private static final String EXTERNAL_VARIABLE_NAME = "EXTVAR1"; + private static final String EXTERNAL_VARIABLE_VALUE = "ext1"; + + @Test + @DisplayName("Should not add variable if not already existant and null in raw") + void shouldNotAddIfNull() { + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with new null variables + Map collectedVariablesMap = new LinkedHashMap<>(); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", COLLECTED_VARIABLE_VALUE)); + collectedVariablesMap.put("VAR2", null); + + Map externalVariablesMap = new LinkedHashMap<>(); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, EXTERNAL_VARIABLE_VALUE); + externalVariablesMap.put("EXTVAR2", null); + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertNonNullVariables(surveyUnitModels); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should convert to non null if existant variable is null or absent") + void shouldConvertToNonNullValueIfNullOrNotExists(boolean isVariableAlreadyPresent){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(QUESTIONNAIRE_ID) + .interrogationId(INTERROGATION_ID) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + if(isVariableAlreadyPresent) { + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + } + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response + Map collectedVariablesMap = new LinkedHashMap<>(); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", COLLECTED_VARIABLE_VALUE)); + + Map externalVariablesMap = new LinkedHashMap<>(); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, EXTERNAL_VARIABLE_VALUE); + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertNonNullVariables(surveyUnitModels); + } + + @Test + @DisplayName("Should convert to null values if non null already exists (one value)") + void shouldConvertNullValueIfNonNullOneValue(){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit with non-null variables + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with null + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, null); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, null); + + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertNullVariables(surveyUnitModels); + } + + @Test + @DisplayName("Should convert to null values if non null already exists (multiple iterations)") + void shouldConvertNullValueIfNonNullMultipleValues(){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit with non-null variables and new iterations + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value("VALUE2") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value("ext2") + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with null second iteration + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + List variablesStrings = new ArrayList<>(); + variablesStrings.add(COLLECTED_VARIABLE_VALUE); + variablesStrings.add(null); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", variablesStrings)); + + variablesStrings = new ArrayList<>(); + variablesStrings.add(EXTERNAL_VARIABLE_VALUE); + variablesStrings.add(null); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, variablesStrings); + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertSecondIterationNull(surveyUnitModels); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should keep null value if variable null or absent") + void shouldKeepNull(boolean isNewVariablesPresent){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit with null + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(QUESTIONNAIRE_ID) + .interrogationId(INTERROGATION_ID) + .state(DataState.COLLECTED) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value(null) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value(null) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + if(isNewVariablesPresent) { + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, null); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, null); + } + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertNullVariables(surveyUnitModels); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should keep null value if variable null or absent (multiple iterations)") + void shouldKeepNullIteration(boolean isNewVariablesPresent){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value(null) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value(null) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(QUESTIONNAIRE_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with null or absent second iteration + Map collectedVariablesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + List collectedVariableValues = new ArrayList<>(); + collectedVariableValues.add(COLLECTED_VARIABLE_VALUE); + + List externalVariableValues = new ArrayList<>(); + externalVariableValues.add(EXTERNAL_VARIABLE_VALUE); + + if(isNewVariablesPresent) { + collectedVariableValues.add(null); + externalVariableValues.add(null); + } + + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", collectedVariableValues)); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, externalVariableValues); + + LunaticJsonRawDataModel lunaticJsonRawDataModel = rawData( + Map.of( + "COLLECTED", collectedVariablesMap, + "EXTERNAL", externalVariablesMap + ) + ); + + List lunaticJsonRawDataModels = new ArrayList<>(List.of(lunaticJsonRawDataModel)); + + //WHEN + List surveyUnitModels = converter.convertRawData( + QUESTIONNAIRE_ID, + lunaticJsonRawDataModels, + variablesMap + ); + + //THEN + assertSecondIterationNull(surveyUnitModels); + } + + //NullVariablesTests UTILS + private SurveyUnitModel getSurveyUnitModel() { + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(QUESTIONNAIRE_ID) + .interrogationId(INTERROGATION_ID) + .state(DataState.COLLECTED) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value(COLLECTED_VARIABLE_VALUE) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value(EXTERNAL_VARIABLE_VALUE) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + return surveyUnitModel; + } + + private void assertNullVariables(List surveyUnitModels) { + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo(COLLECTED_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isNull(); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo(EXTERNAL_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isNull(); + } + + private void assertNonNullVariables(List surveyUnitModels) { + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo(COLLECTED_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isEqualTo(COLLECTED_VARIABLE_VALUE); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo(EXTERNAL_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isEqualTo(EXTERNAL_VARIABLE_VALUE); + } + + private void assertSecondIterationNull(List surveyUnitModels) { + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getCollectedVariables()){ + assertThat(variableModel.iteration()).isIn(1,2); + if(variableModel.iteration().equals(2)){ + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo(COLLECTED_VARIABLE_VALUE); + } + + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getExternalVariables()) { + assertThat(variableModel.iteration()).isIn(1,2); + if (variableModel.iteration().equals(2)) { + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo(EXTERNAL_VARIABLE_VALUE); + } + } + + } + private LunaticJsonRawDataModel rawData(Map data) { - LunaticJsonRawDataModel rawData = mock(LunaticJsonRawDataModel.class); - - when(rawData.questionnaireId()).thenReturn("questionnaire-id"); - when(rawData.mode()).thenReturn(Mode.valueOf("WEB")); - when(rawData.interrogationId()).thenReturn("interrogation-id"); - when(rawData.idUE()).thenReturn("survey-unit-id"); - when(rawData.recordDate()).thenReturn(DATE_TIME); - when(rawData.data()).thenReturn(data); + LunaticJsonRawDataModel rawData = LunaticJsonRawDataModel.builder() + .questionnaireId(QUESTIONNAIRE_ID) + .mode(Mode.WEB) + .interrogationId(INTERROGATION_ID) + .idUE("survey-unit-id") + .recordDate(DATE_TIME) + .data(data) + .build(); when(payloadParser.getValidationDate(rawData)).thenReturn(DATE_TIME); when(payloadParser.getIsCapturedIndirectly(rawData)).thenReturn(false); diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java new file mode 100644 index 000000000..9373e2c92 --- /dev/null +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverterTest.java @@ -0,0 +1,141 @@ +package fr.insee.genesis.domain.converter.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; + +@ExtendWith(MockitoExtension.class) +class RawDataConverterTest { + + @Mock + private SurveyUnitService surveyUnitService; + + @InjectMocks + private final RawDataConverter rawDataConverterTestImpl = new RawDataConverter(surveyUnitService) { + @Override + protected Map> getLastSurveyUnitModels(String questionnaireOrCollectionInstrumentId, List interrogationIds) { + return super.getLastSurveyUnitModels(questionnaireOrCollectionInstrumentId, interrogationIds); + } + }; + + + @Nested + @DisplayName("getLastSurveyUnitModels tests") + class getLastSurveyUnitModelsTests{ + @Test + @DisplayName("Should return map of map with interrogation ids and states as keys") + void getLastSurveyUnitModelsTest() { + //GIVEN + String questionnaireId = "questionnaire"; + Set interrogationIds = Set.of("Interrogation1", "Interrogation2"); + + List surveyUnitModels = new ArrayList<>(); + for(String interrogationId : interrogationIds){ + surveyUnitModels.add( + SurveyUnitModel.builder() + .collectionInstrumentId(questionnaireId) + .state(DataState.COLLECTED) + .interrogationId(interrogationId) + .build() + ); + } + + Mockito.when(surveyUnitService.findLatestByInterrogationIds(questionnaireId, interrogationIds) + ).thenReturn(surveyUnitModels); + + //WHEN + + Map> resultMap = rawDataConverterTestImpl.getLastSurveyUnitModels( + questionnaireId, + interrogationIds.stream().toList() + ); + + + //THEN + //Service call with good arguments + @SuppressWarnings("unchecked") + ArgumentCaptor> setArgumentCaptor = ArgumentCaptor.forClass(Set.class); + Mockito.verify(surveyUnitService, Mockito.times(1)).findLatestByInterrogationIds( + eq(questionnaireId), + setArgumentCaptor.capture() + ); + assertThat(setArgumentCaptor.getValue()).containsExactlyInAnyOrder( + "Interrogation1","Interrogation2" + ); + + //Check resulted map key and values content + assertThat(resultMap).containsOnlyKeys(interrogationIds); + for(String interrogationId : interrogationIds){ + Map surveyUnitModelsOfInterrogationId = resultMap.get(interrogationId); + assertThat(surveyUnitModelsOfInterrogationId).containsOnlyKeys(DataState.COLLECTED); + assertThat( + surveyUnitModelsOfInterrogationId.get(DataState.COLLECTED).getInterrogationId() + ).isEqualTo(interrogationId); + } + } + } + + @Nested + @DisplayName("getValueString() util") + class GetValueStringTests { + + @Test + @DisplayName("Double value strips trailing zeros") + void doubleStripsTrailingZeros() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString(1.50)).isEqualTo("1.5"); + } + + @Test + @DisplayName("Float value strips trailing zeros") + void floatStripsTrailingZeros() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString(1.500f)).isEqualTo("1.5"); + } + + @Test + @DisplayName("Integer value returns plain string") + void integerReturnsPlainString() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString(42)).isEqualTo("42"); + } + + @Test + @DisplayName("String value returns same string") + void stringReturnsItself() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString("hello")).isEqualTo("hello"); + } + + @Test + @DisplayName("Null returns 'null' string") + void nullReturnsNullString() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString(null)).isEqualTo("null"); + } + + @Test + @DisplayName("BigDecimal integer-like double has no decimal point") + void bigDecimalIntegerDouble() { + //WHEN + THEN + assertThat(RawDataConverter.getValueString(3.0)).isEqualTo("3"); + } + } +} \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverterTest.java deleted file mode 100644 index 7f3460d84..000000000 --- a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverterTest.java +++ /dev/null @@ -1,314 +0,0 @@ -package fr.insee.genesis.domain.converter.rawdata; - -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; -import fr.insee.genesis.domain.model.surveyunit.DataState; -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; -import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser; -import fr.insee.modelefiliere.RawResponseDto; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.tuple; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class RawResponseConverterTest { - - @Mock - private RawResponsePayloadParser rawResponsePayloadParser; - - @Test - void shouldReturnNoSurveyUnitWhenRawResponseListIsEmpty() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class); - List emptySurveyUnitModels = new ArrayList<>(); - - List result = converter.convertRawResponseAndCollectEmptyModels( - List.of(), - variablesMap, - emptySurveyUnitModels - ); - - assertThat(result).isEmpty(); - assertThat(emptySurveyUnitModels).isEmpty(); - } - - @Test - void shouldCreateCollectedAndEditedSurveyUnitsFromCollectedPayloadStates() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); - - when(variablesMap.getVariable("FIRST_NAME").getGroupName()).thenReturn(Constants.ROOT_GROUP_NAME); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - collectedVariables( - Map.of( - "COLLECTED", "Alice", - "EDITED", "Alicia" - ) - ), - Map.of() - )); - - List result = converter.convertRawResponse( - List.of(rawResponseModel), - variablesMap - ); - - assertThat(result).hasSize(2); - - assertThat(result) - .extracting(SurveyUnitModel::getState) - .containsExactly(DataState.COLLECTED, DataState.EDITED); - - assertThat(result.get(0).getCollectedVariables()) - .extracting("varId", "value", "iteration", "scope") - .containsExactly(tuple("FIRST_NAME", "Alice", 1, Constants.ROOT_GROUP_NAME)); - - assertThat(result.get(1).getCollectedVariables()) - .extracting("varId", "value", "iteration", "scope") - .containsExactly(tuple("FIRST_NAME", "Alicia", 1, Constants.ROOT_GROUP_NAME)); - } - - @Test - void shouldConvertExternalVariablesOnlyForCollectedSurveyUnit() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); - - when(variablesMap.getVariable("COUNTRY").getGroupName()).thenReturn(Constants.ROOT_GROUP_NAME); - when(variablesMap.getVariable("CHILDREN").getGroupName()).thenReturn("HOUSEHOLD"); - - Map externalVariables = new LinkedHashMap<>(); - externalVariables.put("COUNTRY", "FR"); - externalVariables.put("CHILDREN", List.of("Anna", "", "Paul")); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(), - externalVariables - )); - - List emptySurveyUnitModels = new ArrayList<>(); - - List result = converter.convertRawResponseAndCollectEmptyModels( - List.of(rawResponseModel), - variablesMap, - emptySurveyUnitModels - ); - - assertThat(result).hasSize(1); - - SurveyUnitModel collectedSurveyUnit = result.getFirst(); - - assertThat(collectedSurveyUnit.getState()).isEqualTo(DataState.COLLECTED); - assertThat(collectedSurveyUnit.getCollectedVariables()).isEmpty(); - - assertThat(collectedSurveyUnit.getExternalVariables()) - .extracting("varId", "value", "iteration", "scope") - .containsExactly( - tuple("COUNTRY", "FR", 1, Constants.ROOT_GROUP_NAME), - tuple("CHILDREN", "Anna", 1, "HOUSEHOLD"), - tuple("CHILDREN", "Paul", 3, "HOUSEHOLD") - ); - - assertThat(emptySurveyUnitModels) - .hasSize(1) - .extracting(SurveyUnitModel::getState) - .containsExactly(DataState.EDITED); - } - - @Test - void shouldCollectEmptySurveyUnitsWhenNoVariableExists() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(), - Map.of() - )); - - List emptySurveyUnitModels = new ArrayList<>(); - - List result = converter.convertRawResponseAndCollectEmptyModels( - List.of(rawResponseModel), - variablesMap, - emptySurveyUnitModels - ); - - assertThat(result).isEmpty(); - - assertThat(emptySurveyUnitModels) - .hasSize(2) - .extracting(SurveyUnitModel::getState) - .containsExactly(DataState.COLLECTED, DataState.EDITED); - } - - @Test - void shouldAssignRootScopeWhenVariableIsMissingFromMetadata() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class); - - when(variablesMap.getVariable("UNKNOWN_VARIABLE")).thenReturn(null); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(), - Map.of("UNKNOWN_VARIABLE", "value") - )); - - List result = converter.convertRawResponse( - List.of(rawResponseModel), - variablesMap - ); - - assertThat(result).hasSize(1); - - assertThat(result.getFirst().getExternalVariables()) - .singleElement() - .extracting("scope") - .isEqualTo(Constants.ROOT_GROUP_NAME); - } - - @Test - void shouldSetQuestionnaireStateWhenPayloadContainsValidState() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(), - Map.of() - )); - - when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) - .thenReturn(RawResponseDto.QuestionnaireStateEnum.FINISHED.name()); - - List emptySurveyUnitModels = new ArrayList<>(); - - converter.convertRawResponseAndCollectEmptyModels( - List.of(rawResponseModel), - variablesMap, - emptySurveyUnitModels - ); - - assertThat(emptySurveyUnitModels) - .extracting(SurveyUnitModel::getQuestionnaireState) - .containsOnly(RawResponseDto.QuestionnaireStateEnum.FINISHED); - } - - @Test - void shouldKeepQuestionnaireStateNullWhenPayloadContainsInvalidState() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(), - Map.of() - )); - - when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) - .thenReturn("INVALID_STATE"); - - List emptySurveyUnitModels = new ArrayList<>(); - - converter.convertRawResponseAndCollectEmptyModels( - List.of(rawResponseModel), - variablesMap, - emptySurveyUnitModels - ); - - assertThat(emptySurveyUnitModels) - .extracting(SurveyUnitModel::getQuestionnaireState) - .containsOnlyNulls(); - } - - @Test - void shouldConvertPairwiseCollectedVariables() { - RawResponseConverter converter = new RawResponseConverter(rawResponsePayloadParser); - VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); - - when(variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1)).thenReturn(true); - when(variablesMap.getVariable(Constants.PAIRWISE_PREFIX + 1).getGroupName()).thenReturn("PAIRWISE_GROUP"); - - List> pairwiseValues = List.of( - List.of("1", "", "3"), - List.of() - ); - - RawResponseModel rawResponseModel = rawResponseModel(payloadWith( - Map.of(Constants.PAIRWISES, Map.of("COLLECTED", pairwiseValues)), - Map.of() - )); - - List result = converter.convertRawResponse( - List.of(rawResponseModel), - variablesMap - ); - - SurveyUnitModel collectedSurveyUnit = result.getFirst(); - - assertThat(collectedSurveyUnit.getCollectedVariables()) - .hasSize(2 * (Constants.MAX_LINKS_ALLOWED - 1)); - - assertThat(collectedSurveyUnit.getCollectedVariables()) - .extracting("varId", "value", "iteration", "scope", "parentId") - .contains( - tuple(Constants.PAIRWISE_PREFIX + 1, "1", 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), - tuple(Constants.PAIRWISE_PREFIX + 2, Constants.SAME_AXIS_VALUE, 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), - tuple(Constants.PAIRWISE_PREFIX + 3, "3", 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), - tuple(Constants.PAIRWISE_PREFIX + 1, Constants.NO_PAIRWISE_VALUE, 2, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME) - ); - } - - private RawResponseModel rawResponseModel(Map payload) { - RawResponseModel rawResponseModel = mock(RawResponseModel.class); - - when(rawResponseModel.collectionInstrumentId()).thenReturn("collection-instrument-id"); - when(rawResponseModel.mode()).thenReturn(Mode.valueOf("WEB")); - when(rawResponseModel.interrogationId()).thenReturn("interrogation-id"); - when(rawResponseModel.recordDate()).thenReturn(LocalDateTime.parse("2025-01-01T10:00:00")); - when(rawResponseModel.payload()).thenReturn(payload); - - when(rawResponsePayloadParser.getStringField(rawResponseModel, "majorModelVersion")) - .thenReturn("1"); - when(rawResponsePayloadParser.getStringField(rawResponseModel, "usualSurveyUnitId")) - .thenReturn("survey-unit-id"); - when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) - .thenReturn(null); - when(rawResponsePayloadParser.getValidationDate(rawResponseModel)) - .thenReturn(LocalDateTime.parse("2025-01-02T10:00:00")); - when(rawResponsePayloadParser.getIsCapturedIndirectly(rawResponseModel)) - .thenReturn(false); - - return rawResponseModel; - } - - private static Map payloadWith( - Map collectedVariables, - Map externalVariables - ) { - return Map.of( - "data", Map.of( - "COLLECTED", collectedVariables, - "EXTERNAL", externalVariables - ) - ); - } - - private static Map collectedVariables( - Map states - ) { - return Map.of("FIRST_NAME", states); - } -} \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java new file mode 100644 index 000000000..5e5f1424e --- /dev/null +++ b/src/test/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverterTest.java @@ -0,0 +1,786 @@ +package fr.insee.genesis.domain.converter.rawdata; + +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; +import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.modelefiliere.RawResponseDto; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class RawResponseRawDataConverterTest { + + private static final String COLLECTION_INSTRUMENT_ID = "collection-instrument-id"; + private static final String INTERROGATION_ID = "testInterrogation"; + private static final Mode MODE = Mode.WEB; + + + @Mock + private RawResponsePayloadParser rawResponsePayloadParser; + + @Mock + private SurveyUnitService surveyUnitService; + + @InjectMocks + private RawResponseRawDataConverter rawResponseRawDataConverter; + + @Test + void shouldReturnNoSurveyUnitWhenRawResponseListIsEmpty() { + VariablesMap variablesMap = mock(VariablesMap.class); + List emptySurveyUnitModels = new ArrayList<>(); + + List result = rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + COLLECTION_INSTRUMENT_ID, + List.of(), + variablesMap, + emptySurveyUnitModels + ); + + assertThat(result).isEmpty(); + assertThat(emptySurveyUnitModels).isEmpty(); + } + + @Test + void shouldCreateCollectedAndEditedSurveyUnitsFromCollectedPayloadStates() { + VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); + + when(variablesMap.getVariable("FIRST_NAME").getGroupName()).thenReturn(Constants.ROOT_GROUP_NAME); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + collectedVariables( + Map.of( + "COLLECTED", "Alice", + "EDITED", "Alicia" + ) + ), + Map.of() + )); + + List result = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap + ); + + assertThat(result).hasSize(2); + + assertThat(result) + .extracting(SurveyUnitModel::getState) + .containsExactly(DataState.COLLECTED, DataState.EDITED); + + assertThat(result.get(0).getCollectedVariables()) + .extracting("varId", "value", "iteration", "scope") + .containsExactly(tuple("FIRST_NAME", "Alice", 1, Constants.ROOT_GROUP_NAME)); + + assertThat(result.get(1).getCollectedVariables()) + .extracting("varId", "value", "iteration", "scope") + .containsExactly(tuple("FIRST_NAME", "Alicia", 1, Constants.ROOT_GROUP_NAME)); + } + + @Test + void shouldConvertExternalVariablesOnlyForCollectedSurveyUnit() { + VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); + + when(variablesMap.getVariable("COUNTRY").getGroupName()).thenReturn(Constants.ROOT_GROUP_NAME); + when(variablesMap.getVariable("CHILDREN").getGroupName()).thenReturn("HOUSEHOLD"); + + Map externalVariables = new LinkedHashMap<>(); + externalVariables.put("COUNTRY", "FR"); + externalVariables.put("CHILDREN", List.of("Anna", "", "Paul")); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(), + externalVariables + )); + + List emptySurveyUnitModels = new ArrayList<>(); + + List result = rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap, + emptySurveyUnitModels + ); + + assertThat(result).hasSize(1); + + SurveyUnitModel collectedSurveyUnit = result.getFirst(); + + assertThat(collectedSurveyUnit.getState()).isEqualTo(DataState.COLLECTED); + assertThat(collectedSurveyUnit.getCollectedVariables()).isEmpty(); + + assertThat(collectedSurveyUnit.getExternalVariables()) + .extracting("varId", "value", "iteration", "scope") + .containsExactly( + tuple("COUNTRY", "FR", 1, Constants.ROOT_GROUP_NAME), + tuple("CHILDREN", "Anna", 1, "HOUSEHOLD"), + tuple("CHILDREN", "Paul", 3, "HOUSEHOLD") + ); + + assertThat(emptySurveyUnitModels) + .hasSize(1) + .extracting(SurveyUnitModel::getState) + .containsExactly(DataState.EDITED); + } + + @Test + void shouldCollectEmptySurveyUnitsWhenNoVariableExists() { + VariablesMap variablesMap = mock(VariablesMap.class); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(), + Map.of() + )); + + List emptySurveyUnitModels = new ArrayList<>(); + + List result = rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap, + emptySurveyUnitModels + ); + + assertThat(result).isEmpty(); + + assertThat(emptySurveyUnitModels) + .hasSize(2) + .extracting(SurveyUnitModel::getState) + .containsExactly(DataState.COLLECTED, DataState.EDITED); + } + + @Test + void shouldAssignRootScopeWhenVariableIsMissingFromMetadata() { + VariablesMap variablesMap = mock(VariablesMap.class); + + when(variablesMap.getVariable("UNKNOWN_VARIABLE")).thenReturn(null); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(), + Map.of("UNKNOWN_VARIABLE", "value") + )); + + List result = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap + ); + + assertThat(result).hasSize(1); + + assertThat(result.getFirst().getExternalVariables()) + .singleElement() + .extracting("scope") + .isEqualTo(Constants.ROOT_GROUP_NAME); + } + + @Test + void shouldSetQuestionnaireStateWhenPayloadContainsValidState() { + VariablesMap variablesMap = mock(VariablesMap.class); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(), + Map.of() + )); + + when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) + .thenReturn(RawResponseDto.QuestionnaireStateEnum.FINISHED.name()); + + List emptySurveyUnitModels = new ArrayList<>(); + + rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap, + emptySurveyUnitModels + ); + + assertThat(emptySurveyUnitModels) + .extracting(SurveyUnitModel::getQuestionnaireState) + .containsOnly(RawResponseDto.QuestionnaireStateEnum.FINISHED); + } + + @Test + void shouldKeepQuestionnaireStateNullWhenPayloadContainsInvalidState() { + VariablesMap variablesMap = mock(VariablesMap.class); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(), + Map.of() + )); + + when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) + .thenReturn("INVALID_STATE"); + + List emptySurveyUnitModels = new ArrayList<>(); + + rawResponseRawDataConverter.convertRawResponseAndCollectEmptyModels( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap, + emptySurveyUnitModels + ); + + assertThat(emptySurveyUnitModels) + .extracting(SurveyUnitModel::getQuestionnaireState) + .containsOnlyNulls(); + } + + @Test + void shouldConvertPairwiseCollectedVariables() { + RawResponseRawDataConverter converter = new RawResponseRawDataConverter( + surveyUnitService, rawResponsePayloadParser + ); + VariablesMap variablesMap = mock(VariablesMap.class, RETURNS_DEEP_STUBS); + + when(variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1)).thenReturn(true); + when(variablesMap.getVariable(Constants.PAIRWISE_PREFIX + 1).getGroupName()).thenReturn("PAIRWISE_GROUP"); + + List> pairwiseValues = List.of( + List.of("1", "", "3"), + List.of() + ); + + RawResponseModel rawResponseModel = rawResponseModel(payloadWith( + Map.of(Constants.PAIRWISES, Map.of("COLLECTED", pairwiseValues)), + Map.of() + )); + + List result = converter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + List.of(rawResponseModel), + variablesMap + ); + + SurveyUnitModel collectedSurveyUnit = result.getFirst(); + + assertThat(collectedSurveyUnit.getCollectedVariables()) + .hasSize(2 * (Constants.MAX_LINKS_ALLOWED - 1)); + + assertThat(collectedSurveyUnit.getCollectedVariables()) + .extracting("varId", "value", "iteration", "scope", "parentId") + .contains( + tuple(Constants.PAIRWISE_PREFIX + 1, "1", 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), + tuple(Constants.PAIRWISE_PREFIX + 2, Constants.SAME_AXIS_VALUE, 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), + tuple(Constants.PAIRWISE_PREFIX + 3, "3", 1, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME), + tuple(Constants.PAIRWISE_PREFIX + 1, Constants.NO_PAIRWISE_VALUE, 2, "PAIRWISE_GROUP", Constants.ROOT_GROUP_NAME) + ); + } + + @Nested + @DisplayName("Null cases tests") + class NullVariablesTests { + + //NullVariablesTests constants + private static final String COLLECTED_VARIABLE_NAME = "VAR1"; + private static final String COLLECTED_VARIABLE_VALUE = "test"; + private static final String EXTERNAL_VARIABLE_NAME = "EXTVAR1"; + private static final String EXTERNAL_VARIABLE_VALUE = "ext1"; + + @Test + @DisplayName("Should not add variable if not already existant and null in raw") + void shouldNotAddIfNull(){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with new null variables + Map collectedVariablesMap = new LinkedHashMap<>(); + + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", COLLECTED_VARIABLE_VALUE)); + collectedVariablesMap.put("VAR2", null); + + Map externalVariablesMap = new LinkedHashMap<>(); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, EXTERNAL_VARIABLE_VALUE); + externalVariablesMap.put("EXTVAR2", null); + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + collectedVariablesMap, + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + // No new variables added + assertNonNullVariables(surveyUnitModels); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should convert to non null if existant variable is null or absent") + void shouldConvertToNonNullValueIfNullOrNotExists(boolean addNullVariables){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .interrogationId(INTERROGATION_ID) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + //Add null variables if true + if(addNullVariables) { + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value(null) + .state(DataState.COLLECTED) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + } + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response + Map collectedVariablesMap = new LinkedHashMap<>(); + collectedVariablesMap.put(COLLECTED_VARIABLE_NAME, Map.of("COLLECTED", COLLECTED_VARIABLE_VALUE)); + + Map externalVariablesMap = new LinkedHashMap<>(); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, EXTERNAL_VARIABLE_VALUE); + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + collectedVariablesMap, + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + assertNonNullVariables(surveyUnitModels); + } + + @Test + @DisplayName("Should convert to null values if non null already exists (one value)") + void shouldConvertNullValueIfNonNullOneValue(){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit with non-null variables + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with null + Map collectedVariableStatesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + collectedVariableStatesMap.put("COLLECTED", null); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, null); + + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + Map.of(COLLECTED_VARIABLE_NAME, collectedVariableStatesMap), + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + assertNullVariables(surveyUnitModels); + } + + @Test + @DisplayName("Should convert to null values if non null already exists (multiple iterations)") + void shouldConvertNullValueIfNonNullMultipleValues(){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit with non-null variables + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value("VALUE2") + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value("ext2") + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with null second iteration + Map collectedVariableStatesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + List variablesStrings = new ArrayList<>(); + variablesStrings.add(COLLECTED_VARIABLE_VALUE); + variablesStrings.add(null); + collectedVariableStatesMap.put("COLLECTED", variablesStrings); + + variablesStrings = new ArrayList<>(); + variablesStrings.add(EXTERNAL_VARIABLE_VALUE); + variablesStrings.add(null); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, variablesStrings); + + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + Map.of(COLLECTED_VARIABLE_NAME, collectedVariableStatesMap), + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + assertSecondIterationNull(surveyUnitModels); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should keep null value if variable null or absent in raw") + void shouldKeepNull(boolean isNewVariablesPresent){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit with null variables + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .interrogationId(INTERROGATION_ID) + .state(DataState.COLLECTED) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value(null) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value(null) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with null variables or no variable at all + Map collectedVariableStatesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + if(isNewVariablesPresent) { + collectedVariableStatesMap.put("COLLECTED", null); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, null); + } + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + Map.of(COLLECTED_VARIABLE_NAME, collectedVariableStatesMap), + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + assertNullVariables(surveyUnitModels); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @DisplayName("Should keep null value if variable null or absent (multiple iterations)") + void shouldKeepNullIteration(boolean isSecondIterationPresent){ + //GIVEN + VariablesMap variablesMap = mock(VariablesMap.class); + + //Already existing survey unit with null second iterations + SurveyUnitModel surveyUnitModel = getSurveyUnitModel(); + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value(null) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value(null) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(2) + .build() + ); + + when(surveyUnitService.findLatestByInterrogationIds(eq(COLLECTION_INSTRUMENT_ID), anySet())) + .thenReturn(List.of( + surveyUnitModel + )); + + //Raw response with null second iteration or no second iteration at all + Map collectedVariableStatesMap = new LinkedHashMap<>(); + Map externalVariablesMap = new LinkedHashMap<>(); + + List collectedVariablesValues = new ArrayList<>(); + collectedVariablesValues.add(COLLECTED_VARIABLE_VALUE); + + List externalVariablesValues = new ArrayList<>(); + externalVariablesValues.add(EXTERNAL_VARIABLE_VALUE); + + if(isSecondIterationPresent) { + collectedVariablesValues.add(null); + externalVariablesValues.add(null); + } + + collectedVariableStatesMap.put("COLLECTED", collectedVariablesValues); + externalVariablesMap.put(EXTERNAL_VARIABLE_NAME, externalVariablesValues); + + RawResponseModel rawResponseModel = rawResponseModel( + payloadWith( + Map.of(COLLECTED_VARIABLE_NAME, collectedVariableStatesMap), + externalVariablesMap + ) + ); + + List rawResponseModels = new ArrayList<>(List.of(rawResponseModel)); + + //WHEN + List surveyUnitModels = rawResponseRawDataConverter.convertRawResponse( + COLLECTION_INSTRUMENT_ID, + rawResponseModels, + variablesMap + ); + + //THEN + assertSecondIterationNull(surveyUnitModels); + } + + //NullVariablesTests UTILS + private SurveyUnitModel getSurveyUnitModel() { + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .interrogationId(INTERROGATION_ID) + .state(DataState.COLLECTED) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + surveyUnitModel.getCollectedVariables().add( + VariableModel.builder() + .varId(COLLECTED_VARIABLE_NAME) + .value(COLLECTED_VARIABLE_VALUE) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + surveyUnitModel.getExternalVariables().add( + VariableModel.builder() + .varId(EXTERNAL_VARIABLE_NAME) + .value(EXTERNAL_VARIABLE_VALUE) + .scope(Constants.ROOT_GROUP_NAME) + .iteration(1) + .build() + ); + return surveyUnitModel; + } + + private void assertNullVariables(List surveyUnitModels) { + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo(NullVariablesTests.COLLECTED_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isNull(); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo(NullVariablesTests.EXTERNAL_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isNull(); + } + + private void assertNonNullVariables(List surveyUnitModels) { + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .varId()).isEqualTo(COLLECTED_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getCollectedVariables().getFirst() + .value()).isEqualTo(COLLECTED_VARIABLE_VALUE); + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(1); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .varId()).isEqualTo(EXTERNAL_VARIABLE_NAME); + assertThat(surveyUnitModels.getFirst().getExternalVariables().getFirst() + .value()).isEqualTo(EXTERNAL_VARIABLE_VALUE); + } + + private void assertSecondIterationNull(List surveyUnitModels) { + assertThat(surveyUnitModels).hasSize(1); + assertThat(surveyUnitModels.getFirst().getCollectedVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getCollectedVariables()){ + assertThat(variableModel.iteration()).isIn(1,2); + if(variableModel.iteration().equals(2)){ + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo(COLLECTED_VARIABLE_VALUE); + } + + assertThat(surveyUnitModels.getFirst().getExternalVariables()).hasSize(2); + for(VariableModel variableModel : surveyUnitModels.getFirst().getExternalVariables()) { + assertThat(variableModel.iteration()).isIn(1,2); + if (variableModel.iteration().equals(2)) { + assertThat(variableModel.value()).isNull(); + continue; + } + assertThat(variableModel.value()).isEqualTo(EXTERNAL_VARIABLE_VALUE); + } + } + + } + + + //UTILS + private RawResponseModel rawResponseModel(Map payload) { + RawResponseModel rawResponseModel = RawResponseModel.builder() + .collectionInstrumentId(COLLECTION_INSTRUMENT_ID) + .mode(MODE) + .interrogationId(INTERROGATION_ID) + .recordDate(LocalDateTime.parse("2025-01-01T10:00:00")) + .payload(payload) + .build(); + + when(rawResponsePayloadParser.getStringField(rawResponseModel, "majorModelVersion")) + .thenReturn("1"); + when(rawResponsePayloadParser.getStringField(rawResponseModel, "usualSurveyUnitId")) + .thenReturn("survey-unit-id"); + when(rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState")) + .thenReturn(null); + when(rawResponsePayloadParser.getValidationDate(rawResponseModel)) + .thenReturn(LocalDateTime.parse("2025-01-02T10:00:00")); + when(rawResponsePayloadParser.getIsCapturedIndirectly(rawResponseModel)) + .thenReturn(false); + + return rawResponseModel; + } + + private static Map payloadWith( + Map collectedVariables, + Map externalVariables + ) { + return Map.of( + "data", Map.of( + "COLLECTED", collectedVariables, + "EXTERNAL", externalVariables + ) + ); + } + + private static Map collectedVariables( + Map states + ) { + return Map.of("FIRST_NAME", states); + } +} \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index ce9d62cbf..17c5c6470 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -98,7 +98,7 @@ class LunaticJsonRawDataServiceTest { @BeforeEach void init() { - lunaticJsonRawDataConverter = new LunaticJsonRawDataConverter(new LunaticJsonRawDataPayloadParser()); + lunaticJsonRawDataConverter = new LunaticJsonRawDataConverter(surveyUnitService, new LunaticJsonRawDataPayloadParser()); service = new LunaticJsonRawDataService( controllerUtils, @@ -291,7 +291,9 @@ class ConvertRawDataTests { @DisplayName("Empty raw data list returns empty list") void emptyRawDataList_returnsEmpty() { //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(), new VariablesMap() + ); //THEN assertThat(result).isEmpty(); @@ -307,6 +309,7 @@ void noVariables_rawDataIgnored() { // WHEN List result = lunaticJsonRawDataConverter.convertRawDataAndCollectEmptyModels( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap(), emptySurveyUnitModels @@ -330,7 +333,9 @@ void withCollectedVariable_producesBothDataStates() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(collected); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); //THEN // Expect 1 COLLECTED (VAR1 has a value) and 0 EDITED (EDITED value is null → empty) @@ -355,7 +360,9 @@ void withEditedVariable_producesEditedModel() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(collected); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); //THEN long editedCount = result.stream() @@ -381,7 +388,9 @@ void filiereModelType_detected() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(outerData); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); // Should not throw — FILIERE path is followed TODO More asserts assertThat(result).isNotNull(); @@ -412,7 +421,9 @@ void optionalFields_mappedCorrectly() { .build(); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); //THEN assertThat(result).isNotEmpty(); @@ -447,7 +458,9 @@ void malformedValidationDate_fallsBackToNull() { .build(); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); //THEN assertThat(result).isNotEmpty(); result.stream() @@ -468,7 +481,9 @@ void arrayValues_convertedToMultipleIterations() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(collected); //WHEN - List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData( + QUESTIONNAIRE_ID, List.of(rawData), new VariablesMap() + ); //THEN SurveyUnitModel collectedModel = result.stream() @@ -720,53 +735,6 @@ void returnsPage() { } } - @Nested - @DisplayName("getValueString() util") - class GetValueStringTests { - - @Test - @DisplayName("Double value strips trailing zeros") - void doubleStripsTrailingZeros() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString(1.50)).isEqualTo("1.5"); - } - - @Test - @DisplayName("Float value strips trailing zeros") - void floatStripsTrailingZeros() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString(1.500f)).isEqualTo("1.5"); - } - - @Test - @DisplayName("Integer value returns plain string") - void integerReturnsPlainString() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString(42)).isEqualTo("42"); - } - - @Test - @DisplayName("String value returns same string") - void stringReturnsItself() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString("hello")).isEqualTo("hello"); - } - - @Test - @DisplayName("Null returns 'null' string") - void nullReturnsNullString() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString(null)).isEqualTo("null"); - } - - @Test - @DisplayName("BigDecimal integer-like double has no decimal point") - void bigDecimalIntegerDouble() { - //WHEN + THEN - assertThat(LunaticJsonRawDataService.getValueString(3.0)).isEqualTo("3"); - } - } - //UTILS /** Builds a minimal raw data document with COLLECTED data. */ private LunaticJsonRawDataModel buildRawDataWithCollected(Map collectedData) { diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java index 2ce81f1f1..cc81ecf0d 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java @@ -4,7 +4,7 @@ import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.TestConstants; import fr.insee.genesis.controller.utils.ControllerUtils; -import fr.insee.genesis.domain.converter.rawdata.RawResponseConverter; +import fr.insee.genesis.domain.converter.rawdata.RawResponseRawDataConverter; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; @@ -69,7 +69,7 @@ class RawResponseServiceUnitTest { @Mock private SurveyUnitQualityToolService surveyUnitQualityToolService; - private RawResponseConverter rawResponseConverter; + private RawResponseRawDataConverter rawResponseRawDataConverter; @Mock static DataProcessingContextService dataProcessingContextService; @@ -80,7 +80,7 @@ class RawResponseServiceUnitTest { @BeforeEach void init() { - rawResponseConverter = new RawResponseConverter(new RawResponsePayloadParser()); + rawResponseRawDataConverter = new RawResponseRawDataConverter(surveyUnitService, new RawResponsePayloadParser()); rawResponseService = new RawResponseService( controllerUtils, @@ -90,7 +90,7 @@ void init() { surveyUnitQualityToolService, new FileUtils(TestConstants.getConfigStub()), TestConstants.getConfigStub(), - rawResponseConverter, + rawResponseRawDataConverter, rawResponsePersistencePort ); } @@ -133,7 +133,7 @@ void getUnprocessedCollectionInstrumentIds_shouldnt_return_if_no_spec() { surveyUnitQualityToolService, new FileUtils(TestConstants.getConfigStub()), TestConstants.getConfigStub(), - rawResponseConverter, + rawResponseRawDataConverter, rawResponsePersistencePort ); @@ -493,7 +493,9 @@ void convertRawResponse_shouldConvertCollectedVariable() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID,rawResponses, variablesMap + ); // THEN Assertions.assertThat(result).hasSize(1); // only COLLECTED state (EDITED has no data) @@ -510,7 +512,9 @@ void convertRawResponse_shouldConvertEditedVariable() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, rawResponses, variablesMap + ); // THEN // On attend 1 modèle EDITED avec des variables @@ -530,7 +534,9 @@ void convertRawResponse_shouldConvertExternalVariables() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, rawResponses, variablesMap + ); // THEN List collectedModels = result.stream() @@ -549,7 +555,8 @@ void convertRawResponse_shouldIgnoreEmptyResponse() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, rawResponses, variablesMap); // THEN Assertions.assertThat(result).isEmpty(); @@ -563,7 +570,9 @@ void convertRawResponse_shouldHandleListValues() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, rawResponses, variablesMap + ); // THEN List collectedModels = result.stream() @@ -585,7 +594,9 @@ void convertRawResponse_shouldSkipNullOrEmptyListValues() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseRawDataConverter.convertRawResponse( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, rawResponses, variablesMap + ); // THEN List collectedModels = result.stream() diff --git a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java index 9e7e1e2c6..341f747a3 100644 --- a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java @@ -37,14 +37,19 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; class SurveyUnitServiceTest { @@ -74,7 +79,7 @@ void get_latest_should_return_usualSurveyId() { doReturn(SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(Collections.singletonList(surveyUnitDocument))) .when(surveyUnitPersistencePortStub).findByIds(any(), any()); - List surveyUnitModels = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List surveyUnitModels = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( TestConstants.DEFAULT_INTERROGATION_ID, TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -106,7 +111,7 @@ void get_latest_should_return_state_in_collected_variables(){ .when(surveyUnitPersistencePortStub).findByIds(any(), any()); //WHEN - List surveyUnitModels = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List surveyUnitModels = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( TestConstants.DEFAULT_INTERROGATION_ID, TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -140,7 +145,7 @@ void get_latest_should_return_state_in_external_variables(){ .when(surveyUnitPersistencePortStub).findByIds(any(), any()); //WHEN - List surveyUnitModels = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List surveyUnitModels = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( TestConstants.DEFAULT_INTERROGATION_ID, TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -317,7 +322,7 @@ void get_latest_should_return_usualSurveyId_when_idue() { doReturn(SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(Collections.singletonList(surveyUnitDocument))) .when(surveyUnitPersistencePortStub).findByIds(any(), any()); - List surveyUnitModels = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List surveyUnitModels = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( TestConstants.DEFAULT_INTERROGATION_ID, TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -359,7 +364,7 @@ void get_latest_should_return_edited() { doReturn(SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnitDocuments)) .when(surveyUnitPersistencePortStub).findByIds(any(), any()); - List surveyUnitModels = surveyUnitService.findLatestByIdAndByCollectionInstrumentId( + List surveyUnitModels = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId( TestConstants.DEFAULT_INTERROGATION_ID, TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -781,6 +786,146 @@ private VariablesMap buildVariablesMapWithVar() { } } + @Nested + @DisplayName("findLatestByInterrogationIds tests") + class findLatestByInterrogationIdsTests { + @Test + @DisplayName("Should return empty list if no survey unit is found") + void shouldReturnEmptyList() { + // GIVEN + String questionnaireId = "QUESTIONNAIRE"; + Set interrogationIds = Set.of("INT1", "INT2"); + + when(surveyUnitPersistencePortStub.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + eq(questionnaireId), + anyList())) + .thenReturn(List.of()); + + // WHEN + List result = + surveyUnitService.findLatestByInterrogationIds(questionnaireId, interrogationIds); + + // THEN + assertThat(result).isEmpty(); + + verify(surveyUnitPersistencePortStub) + .findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + eq(questionnaireId), + argThat(ids -> ids.containsAll(interrogationIds) && ids.size() == 2) + ); + } + + @Test + @DisplayName("Should return survey unit (only 1 version)") + void should_return_latest_survey_unit_when_single_version_exists() { + // GIVEN + String questionnaireId = "QUESTIONNAIRE"; + Set interrogationIds = Set.of("INT1"); + + SurveyUnitModel surveyUnit = SurveyUnitModel.builder() + .interrogationId("INT1") + .mode(Mode.WEB) + .recordDate(Instant.now()) + .collectedVariables(new ArrayList<>(List.of( + VariableModel.builder() + .varId("VAR1") + .iteration(1) + .value("VALUE1") + .build()))) + .externalVariables(new ArrayList<>()) + .build(); + + when(surveyUnitPersistencePortStub.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + eq(questionnaireId), + anyList())) + .thenReturn(List.of(surveyUnit)); + + // WHEN + List result = + surveyUnitService.findLatestByInterrogationIds(questionnaireId, interrogationIds); + + // THEN + assertThat(result) + .hasSize(1) + .first() + .extracting(SurveyUnitModel::getInterrogationId) + .isEqualTo("INT1"); + } + + @Test + @DisplayName("Should return survey unit (multiple versions)") + void should_keep_latest_variable_values_when_multiple_versions_exist() { + // GIVEN + String questionnaireId = "QUESTIONNAIRE"; + Set interrogationIds = Set.of("INT1"); + + Instant latestDate = Instant.parse("2025-01-02T10:00:00Z"); + Instant oldestDate = Instant.parse("2025-01-01T10:00:00Z"); + + VariableModel latestVariable = VariableModel.builder() + .varId("VAR1") + .iteration(1) + .value("NEW_VALUE") + .build(); + + VariableModel oldVariable = VariableModel.builder() + .varId("VAR1") + .iteration(1) + .value("OLD_VALUE") + .build(); + + VariableModel oldVariableOnlyInOldVersion = VariableModel.builder() + .varId("VAR2") + .iteration(1) + .value("VALUE2") + .build(); + + SurveyUnitModel latestVersion = SurveyUnitModel.builder() + .interrogationId("INT1") + .mode(Mode.WEB) + .recordDate(latestDate) + .collectedVariables(new ArrayList<>(List.of(latestVariable))) + .externalVariables(new ArrayList<>()) + .build(); + + SurveyUnitModel oldestVersion = SurveyUnitModel.builder() + .interrogationId("INT1") + .mode(Mode.WEB) + .recordDate(oldestDate) + .collectedVariables(new ArrayList<>(List.of(oldVariable, oldVariableOnlyInOldVersion))) + .externalVariables(new ArrayList<>()) + .build(); + + when(surveyUnitPersistencePortStub.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + eq(questionnaireId), + anyList())) + .thenReturn(List.of(oldestVersion, latestVersion)); + + // WHEN + List result = + surveyUnitService.findLatestByInterrogationIds(questionnaireId, interrogationIds); + + // THEN + assertThat(result).hasSize(2); + + SurveyUnitModel latestResult = result.get(0); + SurveyUnitModel additionalVariablesResult = result.get(1); + + assertThat(latestResult.getCollectedVariables()) + .extracting(VariableModel::varId) + .containsExactly("VAR1"); + + assertThat(latestResult.getCollectedVariables()) + .extracting(VariableModel::value) + .containsExactly("NEW_VALUE"); + + assertThat(additionalVariablesResult.getCollectedVariables()) + .extracting(VariableModel::varId) + .containsExactly("VAR2"); + } + + } + //UTILS private SurveyUnitDocument buildDoc(String interrogationId, Mode mode) { diff --git a/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java b/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java index c2df62269..4ef2d2cbb 100644 --- a/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java +++ b/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java @@ -255,59 +255,7 @@ void shouldCorrectInvalidIterationOnFormattedSurveyUnit() { } @ParameterizedTest - @EnumSource(value = DataState.class, names = {"EDITED", "FORCED", "FORMATTED"}, mode = EnumSource.Mode.EXCLUDE) - void shouldCreateFormattedIfNull(DataState dataState) { - String variableName = "varnull"; - for(VariableType variableType : VariableType.values()){ - if(variableType.equals(VariableType.STRING)){ - continue; //Skip STRING - } - variablesMap.removeVariable(variableName); - - // GIVEN - Variable variableDefinition = new Variable( - variableName, - new MetadataModel().getRootGroup(), - variableType - ); - variablesMap.putVariable(variableDefinition); - - // Add null value - surveyUnits.clear(); - VariableModel collectedVariable1 = VariableModel.builder() - .varId(variableName) - .value(null) - .scope(Constants.ROOT_GROUP_NAME) - .iteration(1) - .build(); - SurveyUnitModel surveyUnit = SurveyUnitModel.builder() - .interrogationId("UE1100000001") - .collectionInstrumentId("Quest1") - .state(dataState) - .collectedVariables(List.of(collectedVariable1)) - .externalVariables(List.of()) - .build(); - surveyUnits.add(surveyUnit); - - // WHEN - DataVerifier.verifySurveyUnits(surveyUnits, variablesMap); - - // THEN FORMATTED values added - try{ - Assertions.assertTrue(surveyUnits.size() > 1); - SurveyUnitModel formattedUnit = surveyUnits.get(1); - Assertions.assertEquals(DataState.FORMATTED, formattedUnit.getState()); - Assertions.assertEquals(1, formattedUnit.getCollectedVariables().size()); - Assertions.assertEquals("", formattedUnit.getCollectedVariables().getFirst().value()); // Corrected value - }catch (AssertionFailedError afe){ - log.error("Failed on type {}", variableType); - throw afe; - } - } - } - - @ParameterizedTest - @EnumSource(value = DataState.class, names = {"EDITED", "FORCED", "FORMATTED"}) + @EnumSource(value = DataState.class) void shouldNotCreateFormattedIfNull(DataState dataState) { String variableName = "varnull"; for(VariableType variableType : VariableType.values()){ diff --git a/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapterTest.java b/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapterTest.java index d7aaf8ee4..ab5b18e4a 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapterTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapterTest.java @@ -44,7 +44,6 @@ class SurveyUnitMongoAdapterTest { private static final String QUESTIONNAIRE_ID = "questionnaire-123"; private static final String COLLECTION_INSTRUMENT_ID = "instrument-456"; private static final String INTERROGATION_ID = "interrogation-789"; - private static final String CAMPAIGN_ID = "campaign-101"; private static final String MODE = "CAWI"; @Mock @@ -180,7 +179,7 @@ void findBySetOfIds_shouldReturnMappedModels() { .thenReturn(List.of(buildDoc("i1"))); // WHEN - List result = adapter.findBySetOfIdsAndQuestionnaireIdAndMode(QUESTIONNAIRE_ID, MODE, ids); + List result = adapter.findByQuestionnaireIdAndModeAndInterrogationIds(QUESTIONNAIRE_ID, MODE, ids); // THEN assertThat(result).hasSize(1); @@ -193,13 +192,144 @@ void findBySetOfIds_noResults_shouldReturnEmptyList() { when(mongoRepository.findBySetOfIdsAndQuestionnaireIdAndMode(any(), any(), any())).thenReturn(List.of()); // WHEN - List result = adapter.findBySetOfIdsAndQuestionnaireIdAndMode(QUESTIONNAIRE_ID, MODE, List.of()); + List result = adapter.findByQuestionnaireIdAndModeAndInterrogationIds(QUESTIONNAIRE_ID, MODE, List.of()); // THEN assertThat(result).isEmpty(); } } + @Nested + @DisplayName("findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds() tests") + class FindByCollectionInstrumentIdAndModeAndInterrogationIdsTests { + @Test + @DisplayName("Should return models from both repository methods combined") + void find_shouldCombineBothRepositoryResults() { + // GIVEN + SurveyUnitDocument docByQuestionnaire = buildDoc("INTERRO01"); + SurveyUnitDocument docByCollectionInstrument = buildDoc("INTERRO02"); + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds("QUEST01", List.of("INTERRO01", "INTERRO02"))) + .thenReturn(List.of(docByQuestionnaire)); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds("QUEST01", List.of("INTERRO01", "INTERRO02"))) + .thenReturn(List.of(docByCollectionInstrument)); + + // WHEN + List result = adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01", "INTERRO02")); + + // THEN + assertThat(result).hasSize(2); + } + + @Test + @DisplayName("Should return models from questionnaireId repository only when collectionInstrument returns empty") + void find_onlyQuestionnaireResults_shouldReturnThose() { + // GIVEN + SurveyUnitDocument doc = buildDoc("INTERRO01"); + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds("QUEST01", List.of("INTERRO01"))) + .thenReturn(List.of(doc)); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds("QUEST01", List.of("INTERRO01"))) + .thenReturn(List.of()); + + // WHEN + List result = adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01")); + + // THEN + assertThat(result).hasSize(1); + } + + @Test + @DisplayName("Should return models from collectionInstrumentId repository only when questionnaire returns empty") + void find_onlyCollectionInstrumentResults_shouldReturnThose() { + // GIVEN + SurveyUnitDocument doc = buildDoc("INTERRO01"); + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds("QUEST01", List.of("INTERRO01"))) + .thenReturn(List.of()); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds("QUEST01", List.of("INTERRO01"))) + .thenReturn(List.of(doc)); + + // WHEN + List result = adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01")); + + // THEN + assertThat(result).hasSize(1); + } + + @Test + @DisplayName("Should return empty list when both repositories return empty") + void find_bothEmpty_shouldReturnEmptyList() { + // GIVEN + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + + // WHEN + List result = adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01")); + + // THEN + assertThat(result).isEmpty(); + } + + @Test + @DisplayName("Should pass the same id to both repository methods") + void find_shouldPassSameIdToBothRepositories() { + // GIVEN + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + + // WHEN + adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01")); + + // THEN + verify(mongoRepository).findByQuestionnaireIdAndInterrogationIds(eq("QUEST01"), any()); + verify(mongoRepository).findByCollectionInstrumentIdAndInterrogationIds(eq("QUEST01"), any()); + } + + @Test + @DisplayName("Should pass the same interrogationId list to both repository methods") + void find_shouldPassSameInterrogationIdListToBothRepositories() { + // GIVEN + List interrogationIds = List.of("INTERRO01", "INTERRO02", "INTERRO03"); + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + + // WHEN + adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds("QUEST01", interrogationIds); + + // THEN + verify(mongoRepository).findByQuestionnaireIdAndInterrogationIds(any(), eq(interrogationIds)); + verify(mongoRepository).findByCollectionInstrumentIdAndInterrogationIds(any(), eq(interrogationIds)); + } + + @Test + @DisplayName("Should return mapped models (not raw documents)") + void find_shouldReturnMappedModels() { + // GIVEN + SurveyUnitDocument doc = buildDoc("INTERRO01"); + when(mongoRepository.findByQuestionnaireIdAndInterrogationIds(any(), any())) + .thenReturn(List.of(doc)); + when(mongoRepository.findByCollectionInstrumentIdAndInterrogationIds(any(), any())) + .thenReturn(List.of()); + + // WHEN + List result = adapter.findByCollectionInstrumentOrQuestionnaireIdAndInterrogationIds( + "QUEST01", List.of("INTERRO01")); + + // THEN + assertThat(result).hasSize(1); + assertThat(result.getFirst().getInterrogationId()).isEqualTo("INTERRO01"); + } + } + @Nested @DisplayName("findByInterrogationId() tests") class FindByInterrogationIdTests {