ISSUE-743 # Support external CSV header referencing and add withHeaders

core/src/main/java/org/jsmart/zerocode/core/domain/Parameterized.java

@@ -19,14 +19,14 @@
 public class Parameterized {
     private final List<Object> valueSource;
     private final List<String> csvSource;
-    private final Boolean ignoreHeader;
+    private final Boolean withHeaders;
 
     public Parameterized(
             @JsonProperty("valueSource") List<Object> valueSource,
             @JsonProperty("csvSource") JsonNode csvSourceJsonNode,
-            @JsonProperty("ignoreHeader") Boolean ignoreHeader) {
+            @JsonProperty("withHeaders") Boolean withHeaders) {
         this.valueSource = valueSource;
-        this.ignoreHeader = Optional.ofNullable(ignoreHeader).orElse(false);
+        this.withHeaders = Optional.ofNullable(withHeaders).orElse(false);
         this.csvSource = Optional.ofNullable(csvSourceJsonNode).map(this::getCsvSourceFrom).orElse(Collections.emptyList());
     }
 
@@ -65,16 +65,16 @@ private List<String> readCsvSourceFromExternalCsvFile(JsonNode csvSourceJsonNode
             List<String> csvSourceFileLines = Files.lines(path)
                     .filter(StringUtils::isNotBlank)
                     .collect(Collectors.toList());
-            if (this.ignoreHeader) {
-                return csvSourceFileLines.stream()
-                        .skip(1)
-                        .collect(Collectors.toList());
-            }
+
             return csvSourceFileLines;
         }
         return Collections.emptyList();
     }
 
+    public Boolean isWithHeaders() {
+        return withHeaders;
+    }
+
     @Override
     public String toString() {
         return "Parameterized{" +

core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeParameterizedProcessorImpl.java

@@ -15,6 +15,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.IntStream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static org.jsmart.zerocode.core.constants.ZerocodeConstants.DSL_FORMAT;
 import static org.jsmart.zerocode.core.di.provider.CsvParserProvider.LINE_SEPARATOR;
@@ -116,7 +118,7 @@ private ScenarioSpec resolveParamsCsv(ScenarioSpec scenario, int paramIndex) {
                 return scenario;
             }
 
-            String[] headers = retrieveCsvHeaders(parameterizedCsvList.get(0));
+            String[] headers = retrieveCsvHeaders(parameterizedCsvList.get(0),scenario.getParameterized().isWithHeaders());
 
             paramIndex = headers == null ? paramIndex : paramIndex+1;
 
@@ -126,17 +128,29 @@ private ScenarioSpec resolveParamsCsv(ScenarioSpec scenario, int paramIndex) {
 
             String resultantStepJson = replaceWithValues(stepJson, valuesMap);
 
+            ensureAllParamsResolved(resultantStepJson, headers);
+
             return objectMapper.readValue(resultantStepJson, ScenarioSpec.class);
 
         } catch (Exception exx) {
             throw new RuntimeException("Error while resolving parameterizedCsv values - " + exx);
         }
     }
 
-    private String[] retrieveCsvHeaders(String csvHeaderLine) {
+    private String[] retrieveCsvHeaders(String csvHeaderLine, Boolean isWithHeaders) {
         String[] parsedHeaderLine = csvParser.parseLine(csvHeaderLine + LINE_SEPARATOR);
-        boolean hasHeader = parsedHeaderLine.length > 0 && Arrays.stream(parsedHeaderLine).allMatch(s -> s.matches("^\\|.*\\|$"));
-        return !hasHeader ? null : Arrays.stream(parsedHeaderLine).map(s -> s.substring(1,s.length()-1)).toArray(String[]::new);
+        if (isWithHeaders && parsedHeaderLine.length > 0) {
+            boolean hasPipes = Arrays.stream(parsedHeaderLine).allMatch(s -> s.matches("^\\|.*\\|$"));
+            if (hasPipes) {
+                LOGGER.warn("DEPRECATION WARNING: The '||' (piped) syntax in CSV headers is deprecated. " +
+                "Please stop using it and start using standard header syntax with 'withHeaders: true' field instead. " +
+                "Visit the documentation for more details.");
+                return Arrays.stream(parsedHeaderLine).map(s -> s.substring(1,s.length()-1)).toArray(String[]::new);
+            }
+            return parsedHeaderLine;
+        } else {
+            return null;
+        }
     }
 
     private Map<String, Object> resolveCsvLine(String csvLine, String[] headers) {
@@ -159,4 +173,19 @@ private String replaceWithValues(String stepJson, Map<String, Object> valuesMap)
         return sub.replace(stepJson);
     }
 
+    private void ensureAllParamsResolved(String resultantStepJson, String[] availableHeaders) {
+        Pattern pattern = Pattern.compile("\\$\\{PARAM\\.([^}]+)\\}");
+        Matcher matcher = pattern.matcher(resultantStepJson);
+
+        if (matcher.find()) {
+            String msg;
+            if (availableHeaders == null || availableHeaders.length == 0) {
+                msg = "No available headers found. Ensure your CSV file is not empty and that '\"withHeaders\": true' is defined in the scenario.";
+            } else {
+                msg = "Available headers are: [" + String.join(", ", availableHeaders) + "].";
+            }
+            throw new RuntimeException("Invalid CSV header referenced in scenario. The column '" + matcher.group(1) + "' does not exist. " + msg);
+
+        }
+    }
 }

core/src/main/java/org/jsmart/zerocode/core/runner/ZeroCodeMultiStepsScenarioRunnerImpl.java

@@ -565,8 +565,7 @@ private int getParameterSize(Parameterized parameterized) {
         int csvSourceSize = 0;
 
         if (csvSource != null && !csvSource.isEmpty()){
-            String[] parsedHeaderLine = csvParser.parseLine(csvSource.get(0) + LINE_SEPARATOR);
-            boolean hasHeader = parsedHeaderLine.length > 0 && Arrays.stream(parsedHeaderLine).allMatch(s -> s.matches("^\\|.*\\|$"));
+            Boolean hasHeader = parameterized.isWithHeaders();
             csvSourceSize = hasHeader ? csvSource.size() -1 : csvSource.size();
         }
 

core/src/test/java/org/jsmart/zerocode/core/domain/ParameterizedTest.java

@@ -52,7 +52,7 @@ public void testSerDe_valueSource() throws Exception {
         assertThat(parameterized.getValueSource(), hasItem(true));
 
         String actualJson = mapper.writeValueAsString(parameterized);
-        assertThat(actualJson, is("{\"valueSource\":[\"hello\",123,true],\"csvSource\":[\"1,        2,        200\",\"11,      22,        400\"]}"));
+        assertThat(actualJson, is("{\"valueSource\":[\"hello\",123,true],\"csvSource\":[\"1,        2,        200\",\"11,      22,        400\"],\"withHeaders\":false}"));
     }
 
     @Test
@@ -79,19 +79,4 @@ public void shouldReadCsvSourceFromCsvFile() throws IOException {
         assertThat(parameterized.getCsvSource(), hasItem("siddhagalaxy,Sidd,UK,33847730"));
     }
 
-    @Test
-    public void shouldReadCsvSourceFromCsvFileIgnoringHeader() throws IOException {
-        //given
-        String jsonDocumentAsString =
-                smartUtils.getJsonDocumentAsString("unit_test_files/engine_unit_test_jsons/08.2_parameterized_csv_source_from_file_containing_header.json");
-
-        //when
-        Parameterized parameterized = mapper.readValue(jsonDocumentAsString, Parameterized.class);
-
-        //then
-        assertThat(parameterized.getCsvSource(), hasItem("octocat,The Octocat,San Francisco,583231"));
-        assertThat(parameterized.getCsvSource(), hasItem("siddhagalaxy,Sidd,UK,33847730"));
-        assertThat(parameterized.getCsvSource(), everyItem(not(is("user,name,city,userid"))));//assert header is ignored
-    }
-
 }

core/src/test/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeParameterizedProcessorImplTest.java

@@ -108,4 +108,38 @@ public void testProcessParameterized_csv_with_named_random() throws Exception {
         assertThat(scenarioSpecResolved.getSteps().get(0).getUrl(), is("/anUrl/11/22"));
         assertThat(scenarioSpecResolved.getSteps().get(0).getAssertions().get("status").asInt(), is(400));
     }
+
+    @Test
+    public void testProcessParameterized_csv_with_headers() throws Exception {
+        String jsonDocumentAsString = smartUtils
+                .getJsonDocumentAsString("unit_test_files/engine_unit_test_jsons/11.3_scenario_parameterized_csv_with_headers.json");
+        ScenarioSpec scenarioSpec = mapper.readValue(jsonDocumentAsString, ScenarioSpec.class);
+
+        ScenarioSpec scenarioSpecResolved = parameterizedProcessor.resolveParameterized(scenarioSpec, 0);
+
+
+        assertThat(scenarioSpecResolved.getSteps().get(0).getUrl(), is("/users/1"));
+        assertThat(scenarioSpecResolved.getSteps().get(0).getRequest().get("body").get("name").asText(), is("octocat"));
+    }
+
+   @Test
+    public void testProcessParameterized_csv_with_invalid_header_throws_exception() throws Exception {
+        String jsonDocumentAsString = smartUtils
+                .getJsonDocumentAsString("unit_test_files/engine_unit_test_jsons/11.4_scenario_parameterized_csv_with_invalid_header.json");
+        ScenarioSpec scenarioSpec = mapper.readValue(jsonDocumentAsString, ScenarioSpec.class);
+
+
+        try {
+            parameterizedProcessor.resolveParameterized(scenarioSpec, 0);
+
+            org.junit.Assert.fail("Expected RuntimeException was not thrown for missing CSV header");
+
+        } catch (RuntimeException ex) {
+            org.junit.Assert.assertTrue("Exception message should contain the missing column name", 
+                    ex.getMessage().contains("The column 'typoHere' does not exist"));
+
+            org.junit.Assert.assertTrue("Message should contain the available headers", 
+                    ex.getMessage().contains("Available headers are: [id, name]."));
+        }
+    }
 }

core/src/test/java/org/jsmart/zerocode/core/utils/SmartUtilsTest.java

@@ -76,7 +76,7 @@ public void willGetJsonFileIntoA_JavaString() throws Exception {
     @Test
     public void willReadAllfileNamesFrom_TestResource() {
         List<String> allTestCaseFiles = SmartUtils.getAllEndPointFiles("unit_test_files/engine_unit_test_jsons");
-        assertThat(allTestCaseFiles.size(), is(23));
+        assertThat(allTestCaseFiles.size(), is(25));
         assertThat(allTestCaseFiles.get(0), is("unit_test_files/engine_unit_test_jsons/00_test_json_single_step_verifications.json"));
     }
 

core/src/test/java/org/jsmart/zerocode/parameterized/ParameterisedCsvDemoTest.java

@@ -19,4 +19,9 @@ public void testParameterizedCsv() throws Exception {
     @JsonTestCase("integration_test_files/parameterized/parameterized_sample_csv_with_headers_test.json")
     public void testParameterizedCsvWithHeaders() throws Exception {
     }
+
+    @Test 
+    @JsonTestCase("integration_test_files/parameterized/parameterized_sample_external_csv_with_headers.json")
+    public void testPrameterizedExternalCsvWithHeadersNoPipes() throws Exception {
+    }
 }

core/src/test/resources/integration_test_files/parameterized/parameterized_sample_csv_with_headers_test.json

@@ -20,6 +20,7 @@
         }
     ],
     "parameterized": {
+        "withHeaders" : true,
         "csvSource":[
             "|id|,              |AddressId|,        |status|",
             "${RANDOM.NUMBER},  ${RANDOM.NUMBER},   200",

core/src/test/resources/integration_test_files/parameterized/parameterized_sample_external_csv_with_headers.json

@@ -0,0 +1,26 @@
+{
+    "scenarioName": "Parameterized test scenario demo for external csv",
+    "steps": [
+        {
+            "name": "get_user",
+            "url": "",
+            "operation": "",
+            "request": {
+                "status": "${PARAM.status}",
+                "body": {
+                    "login": "octocat-${PARAM.id}-${PARAM.AddressId}"
+                }
+            },
+            "assertions": {
+                "status": "${$.get_user.response.status}",
+                "body": {
+                    "login": "octocat-${PARAM.id}-${PARAM.AddressId}"
+                }
+            }
+        }
+    ],
+    "parameterized": {
+        "withHeaders" : true,
+        "csvSource": "integration_test_files/parameterized/params_with_headers.csv"
+    }
+}

core/src/test/resources/integration_test_files/parameterized/params_with_headers.csv

@@ -0,0 +1,3 @@
+id, AddressId, status
+12345, 67890, 200
+11, 22, 400

core/src/test/resources/unit_test_files/engine_unit_test_jsons/11.2_scenario_parameterized_csv_with_named_random.json

@@ -20,6 +20,7 @@
         }
     ],
     "parameterized": {
+        "withHeaders": true,
         "csvSource": [
             "|id|,                  |AddressId|,              |status|",
             "${RANDOM.NUMBER},      ${RANDOM.NUMBER},           200",

core/src/test/resources/unit_test_files/engine_unit_test_jsons/11.3_scenario_parameterized_csv_with_headers.json

@@ -0,0 +1,27 @@
+{
+    "scenarioName": "Csv with Headers",
+    "ignoreStepFailures": true,
+    "steps": [
+        {
+            "name": "get_user_details",
+      "url": "/users/${PARAM.userId}",
+      "method": "GET",
+      "request": {
+        "body": {
+          "name": "${PARAM.userName}"
+        }
+            },
+            "assertions": {
+                "status": 200
+            }
+        }
+    ],
+    "parameterized": {
+        "withHeaders": true,
+        "csvSource": [
+        "userId, userName",
+        "1,      octocat",
+        "2,      googlebot"
+        ]
+    }
+}

core/src/test/resources/unit_test_files/engine_unit_test_jsons/11.4_scenario_parameterized_csv_with_invalid_header.json

@@ -0,0 +1,21 @@
+{
+    "scenarioName": "Test Exception on missing header",
+    "steps": [
+        {
+            "name": "get_user",
+            "url": "/users/${PARAM.typoHere}",
+            "request": {
+                "body": {
+                    "name": "octocat"
+                }
+            }
+        }
+    ],
+    "parameterized": {
+        "withHeaders": true,
+        "csvSource": [
+            "id, name",
+            "1, octocat"
+        ]
+    }
+}

http-testing-examples/src/test/java/org/jsmart/zerocode/testhelp/tests/helloworldparameterizedcsv/HelloWorldParameterizedCsvTest.java

@@ -21,8 +21,8 @@ public void testGetByUserNames_csvSourceFiles() throws Exception {
     }
 
     @Test
-    @Scenario("parameterized_csv/hello_world_test_parameterized_csv_source_file_ignore_header.json")
-    public void testGetByUserNames_csvSourceFiles_ignoringHeader() throws Exception {
+    @Scenario("parameterized_csv/hello_world_test_parameterized_csv_source_file_standard_headers.json")
+    public void testGetByUserNames_csvSourceFiles_standard_headers() throws Exception {
     }
 
 }

http-testing-examples/src/test/resources/parameterized_csv/hello_world_test_named_parameterized.json

@@ -21,6 +21,7 @@
         }
     ],
     "parameterized": {
+        "withHeaders": true,
         "csvSource":[
             "|USERID|,     |USERNAME|,  |ADDRESS|,     |USERNO|,        |ISADMIN|",
             "octocat,      The Octocat, San Francisco, 583231,          false",

http-testing-examples/src/test/resources/parameterized_csv/hello_world_test_parameterized_csv_source_file_ignore_header.json → http-testing-examples/src/test/resources/parameterized_csv/hello_world_test_parameterized_csv_source_file_standard_headers.json

@@ -20,7 +20,7 @@
         }
     ],
     "parameterized": {
-        "ignoreHeader": true,
+        "withHeaders": true,
         "csvSource":"parameterized_csv/params_with_header.csv"
     }
 }