diff --git a/launchable/test_runners/pytest.py b/launchable/test_runners/pytest.py index 3341628ff..8639b5d23 100644 --- a/launchable/test_runners/pytest.py +++ b/launchable/test_runners/pytest.py @@ -3,6 +3,7 @@ import os import pathlib import subprocess +from enum import Enum from typing import Generator, List import click @@ -125,6 +126,52 @@ def _pytest_formatter(test_path): split_subset = launchable.CommonSplitSubsetImpls(__name__, formatter=_pytest_formatter).split_subset() +class ReportKind(Enum): + JSON = 0 + XML = 1 + + +def _parse_markers(props: List, kind: ReportKind) -> List[dict]: + """ + Parse marker properties from either XML properties or JSON user_properties format. + """ + markers = [] + marker = {} + + for prop in props: + if kind == ReportKind.JSON: + # JSON format: prop is a list like ["name", "value"] + if not isinstance(prop, list) or len(prop) != 2: + continue + prop_name, prop_value = prop[0], prop[1] + else: + # XML format: prop has .name and .value attributes + prop_name, prop_value = prop.name, prop.value + + if prop_name == "name": + marker["name"] = prop_value + elif prop_name == "args": + if isinstance(prop_value, str): + if prop_value not in ("()", "{}"): + marker["value"] = prop_value + else: + if prop_value not in ((), {}): + marker["value"] = json.dumps(prop_value) + elif prop_name == "kwargs": + if isinstance(prop_value, str): + if prop_value not in ("()", "{}"): + marker["value"] = prop_value + else: + if prop_value not in ((), {}): + marker["value"] = json.dumps(prop_value) + if "value" not in marker: + marker["value"] = "" + markers.append(marker) + marker = {} + + return markers + + @click.option('--json', 'json_report', help="use JSON report files produced by pytest-dev/pytest-reportlog", is_flag=True) @click.argument('source_roots', required=True, nargs=-1) @@ -148,7 +195,7 @@ def data_builder(case: TestCase): ``` """ - markers = [{"name": prop.name, "value": prop.value} for prop in props] + markers = _parse_markers(props, kind=ReportKind.XML) result["markers"] = markers if markers else [] metadata = MetadataTestCase.fromelem(case) @@ -312,15 +359,7 @@ def parse_func( """ props = data.get('user_properties') if isinstance(props, list): - markers = [] - for prop in props: - if isinstance(prop, list) and len(prop) == 2: - # prop is like ["name", "value"] - # prop[0] is name, prop[1] is value - if isinstance(prop[1], str): - markers.append({"name": prop[0], "value": prop[1]}) - else: - markers.append({"name": prop[0], "value": json.dumps(prop[1])}) + markers = _parse_markers(props, kind=ReportKind.JSON) if len(props) > 0: props = {'markers': markers} else: diff --git a/tests/data/pytest/record_test_result.json b/tests/data/pytest/record_test_result.json index ba790eb71..6dbfadf58 100644 --- a/tests/data/pytest/record_test_result.json +++ b/tests/data/pytest/record_test_result.json @@ -64,11 +64,7 @@ "stdout": "", "stderr": "", "data": { - "markers": [ - { "name": "name", "value": "foo" }, - { "name": "args", "value": "()" }, - { "name": "kwargs", "value": "{}" } - ], + "markers": [{ "name": "foo", "value": "" }], "lineNumber": 3 } }, @@ -84,11 +80,7 @@ "stdout": "", "stderr": "@pytest.mark.bar\n def test_func2():\n > assert 1 == False # noqa: E712\n E assert 1 == False\n\n tests/test_funcs1.py:9: AssertionError", "data": { - "markers": [ - { "name": "name", "value": "bar" }, - { "name": "args", "value": "()" }, - { "name": "kwargs", "value": "{}" } - ], + "markers": [{ "name": "bar", "value": "" }], "lineNumber": 7 } }, @@ -105,12 +97,8 @@ "stderr": "x = 0, y = 2\n\n @pytest.mark.parametrize(\"x\", [0, 1])\n @pytest.mark.parametrize(\"y\", [2, 3])\n def test_foo(x, y):\n > assert x == 1\n E assert 0 == 1\n\n tests/test_funcs1.py:15: AssertionError", "data": { "markers": [ - { "name": "name", "value": "parametrize" }, - { "name": "args", "value": "('y', [2, 3])" }, - { "name": "kwargs", "value": "{}" }, - { "name": "name", "value": "parametrize" }, - { "name": "args", "value": "('x', [0, 1])" }, - { "name": "kwargs", "value": "{}" } + { "name": "parametrize", "value": "('x', [0, 1])" }, + { "name": "parametrize", "value": "('y', [2, 3])" } ], "lineNumber": 12 } @@ -128,12 +116,8 @@ "stderr": "", "data": { "markers": [ - { "name": "name", "value": "parametrize" }, - { "name": "args", "value": "('y', [2, 3])" }, - { "name": "kwargs", "value": "{}" }, - { "name": "name", "value": "parametrize" }, - { "name": "args", "value": "('x', [0, 1])" }, - { "name": "kwargs", "value": "{}" } + { "name": "parametrize", "value": "('x', [0, 1])" }, + { "name": "parametrize", "value": "('y', [2, 3])" } ], "lineNumber": 12 } @@ -151,12 +135,8 @@ "stderr": "x = 0, y = 3\n\n @pytest.mark.parametrize(\"x\", [0, 1])\n @pytest.mark.parametrize(\"y\", [2, 3])\n def test_foo(x, y):\n > assert x == 1\n E assert 0 == 1\n\n tests/test_funcs1.py:15: AssertionError", "data": { "markers": [ - { "name": "name", "value": "parametrize" }, - { "name": "args", "value": "('y', [2, 3])" }, - { "name": "kwargs", "value": "{}" }, - { "name": "name", "value": "parametrize" }, - { "name": "args", "value": "('x', [0, 1])" }, - { "name": "kwargs", "value": "{}" } + { "name": "parametrize", "value": "('x', [0, 1])" }, + { "name": "parametrize", "value": "('y', [2, 3])" } ], "lineNumber": 12 } @@ -174,12 +154,8 @@ "stderr": "", "data": { "markers": [ - { "name": "name", "value": "parametrize" }, - { "name": "args", "value": "('y', [2, 3])" }, - { "name": "kwargs", "value": "{}" }, - { "name": "name", "value": "parametrize" }, - { "name": "args", "value": "('x', [0, 1])" }, - { "name": "kwargs", "value": "{}" } + { "name": "parametrize", "value": "('x', [0, 1])" }, + { "name": "parametrize", "value": "('y', [2, 3])" } ], "lineNumber": 12 } @@ -196,11 +172,7 @@ "stdout": "", "stderr": "", "data": { - "markers": [ - { "name": "name", "value": "foo" }, - { "name": "args", "value": "()" }, - { "name": "kwargs", "value": "{}" } - ], + "markers": [{ "name": "foo", "value": "" }], "lineNumber": 3 } }, @@ -216,11 +188,7 @@ "stdout": "", "stderr": "", "data": { - "markers": [ - { "name": "name", "value": "bar" }, - { "name": "args", "value": "()" }, - { "name": "kwargs", "value": "{}" } - ], + "markers": [{ "name": "bar", "value": "" }], "lineNumber": 8 } } diff --git a/tests/data/pytest/record_test_result_json.json b/tests/data/pytest/record_test_result_json.json index e87897ea9..b4de9a7ed 100644 --- a/tests/data/pytest/record_test_result_json.json +++ b/tests/data/pytest/record_test_result_json.json @@ -20,7 +20,18 @@ "status": 1, "stdout": "", "stderr": "", - "data": null + "data": { + "markers": [ + { + "name": "parametrize", + "value": "[\"x\", [0, 1]]" + }, + { + "name": "slow", + "value": "[]" + } + ] + } }, { "type": "case", @@ -64,7 +75,14 @@ "status": 1, "stdout": "", "stderr": "", - "data": null + "data": { + "markers": [ + { + "name": "dependency", + "value": "{\"name\": \"test1\", \"depends\": [\"test0\"]}" + } + ] + } }, { "type": "case", @@ -108,7 +126,14 @@ "status": 1, "stdout": "", "stderr": "", - "data": null + "data": { + "markers": [ + { + "name": "order", + "value": "[2]" + } + ] + } }, { "type": "case", diff --git a/tests/data/pytest/report.json b/tests/data/pytest/report.json index 9add193f4..df94898fa 100644 --- a/tests/data/pytest/report.json +++ b/tests/data/pytest/report.json @@ -6,19 +6,19 @@ {"nodeid": "tests/data/pytest/tests/fooo/func4_test.py", "outcome": "passed", "longrepr": null, "result": null, "sections": [], "$report_type": "CollectReport"} {"nodeid": "tests/data/pytest/tests/fooo/__init__.py", "outcome": "passed", "longrepr": null, "result": null, "sections": [], "$report_type": "CollectReport"} {"nodeid": "tests/data/pytest/tests/funcs3_test.py::test_func4", "location": ["tests/data/pytest/tests/funcs3_test.py", 0, "test_func4"], "keywords": {"tests/data/pytest/tests/funcs3_test.py": 1, "test_func4": 1, "cli": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.05127562699999999, "$report_type": "TestReport"} -{"nodeid": "tests/data/pytest/tests/funcs3_test.py::test_func4", "location": ["tests/data/pytest/tests/funcs3_test.py", 0, "test_func4"], "keywords": {"tests/data/pytest/tests/funcs3_test.py": 1, "test_func4": 1, "cli": 1}, "outcome": "passed", "longrepr": null, "when": "call", "user_properties": [], "sections": [], "duration": 0.001, "$report_type": "TestReport"} +{"nodeid": "tests/data/pytest/tests/funcs3_test.py::test_func4", "location": ["tests/data/pytest/tests/funcs3_test.py", 0, "test_func4"], "keywords": {"tests/data/pytest/tests/funcs3_test.py": 1, "test_func4": 1, "cli": 1}, "outcome": "passed", "longrepr": null, "when": "call", "user_properties": [["name", "parametrize"], ["args", ["x", [0, 1]]], ["kwargs", {}], ["name", "slow"], ["args", []], ["kwargs", {}]], "sections": [], "duration": 0.001, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/funcs3_test.py::test_func4", "location": ["tests/data/pytest/tests/funcs3_test.py", 0, "test_func4"], "keywords": {"tests/data/pytest/tests/funcs3_test.py": 1, "test_func4": 1, "cli": 1}, "outcome": "passed", "longrepr": null, "when": "teardown", "user_properties": [], "sections": [], "duration": 0.001, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/funcs3_test.py::test_func5", "location": ["tests/data/pytest/tests/funcs3_test.py", 4, "test_func5"], "keywords": {"tests/data/pytest/tests/funcs3_test.py": 1, "test_func5": 1, "cli": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.00023489200000004207, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/funcs3_test.py::test_func5", "location": ["tests/data/pytest/tests/funcs3_test.py", 4, "test_func5"], "keywords": {"tests/data/pytest/tests/funcs3_test.py": 1, "test_func5": 1, "cli": 1}, "outcome": "failed", "longrepr": {"reprcrash": {"path": "/Users/yabuki-ryosuke/src/github.com/launchableinc/cli/tests/data/pytest/tests/funcs3_test.py", "lineno": 6, "message": "assert 1 == False"}, "reprtraceback": {"reprentries": [{"type": "ReprEntry", "data": {"lines": [" def test_func5():", "> assert 1 == False", "E assert 1 == False"], "reprfuncargs": {"args": []}, "reprlocals": null, "reprfileloc": {"path": "tests/funcs3_test.py", "lineno": 6, "message": "AssertionError"}, "style": "long"}}], "extraline": null, "style": "long"}, "sections": [], "chain": [[{"reprentries": [{"type": "ReprEntry", "data": {"lines": [" def test_func5():", "> assert 1 == False", "E assert 1 == False"], "reprfuncargs": {"args": []}, "reprlocals": null, "reprfileloc": {"path": "tests/funcs3_test.py", "lineno": 6, "message": "AssertionError"}, "style": "long"}}], "extraline": null, "style": "long"}, {"path": "/Users/yabuki-ryosuke/src/github.com/launchableinc/cli/tests/data/pytest/tests/funcs3_test.py", "lineno": 6, "message": "assert 1 == False"}, null]]}, "when": "call", "user_properties": [], "sections": [], "duration": 0.001, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/funcs3_test.py::test_func5", "location": ["tests/data/pytest/tests/funcs3_test.py", 4, "test_func5"], "keywords": {"tests/data/pytest/tests/funcs3_test.py": 1, "test_func5": 1, "cli": 1}, "outcome": "passed", "longrepr": null, "when": "teardown", "user_properties": [], "sections": [], "duration": 0.00021273599999993564, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/test_funcs1.py::test_func1", "location": ["tests/data/pytest/tests/test_funcs1.py", 0, "test_func1"], "keywords": {"test_func1": 1, "cli": 1, "tests/data/pytest/tests/test_funcs1.py": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.00019361299999998138, "$report_type": "TestReport"} -{"nodeid": "tests/data/pytest/tests/test_funcs1.py::test_func1", "location": ["tests/data/pytest/tests/test_funcs1.py", 0, "test_func1"], "keywords": {"test_func1": 1, "cli": 1, "tests/data/pytest/tests/test_funcs1.py": 1}, "outcome": "passed", "longrepr": null, "when": "call", "user_properties": [], "sections": [], "duration": 0.001, "$report_type": "TestReport"} +{"nodeid": "tests/data/pytest/tests/test_funcs1.py::test_func1", "location": ["tests/data/pytest/tests/test_funcs1.py", 0, "test_func1"], "keywords": {"test_func1": 1, "cli": 1, "tests/data/pytest/tests/test_funcs1.py": 1}, "outcome": "passed", "longrepr": null, "when": "call", "user_properties": [["name", "dependency"], ["args", []], ["kwargs", {"name": "test1", "depends": ["test0"]}]], "sections": [], "duration": 0.001, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/test_funcs1.py::test_func1", "location": ["tests/data/pytest/tests/test_funcs1.py", 0, "test_func1"], "keywords": {"test_func1": 1, "cli": 1, "tests/data/pytest/tests/test_funcs1.py": 1}, "outcome": "passed", "longrepr": null, "when": "teardown", "user_properties": [], "sections": [], "duration": 0.0001104009999999267, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/test_funcs1.py::test_func2", "location": ["tests/data/pytest/tests/test_funcs1.py", 4, "test_func2"], "keywords": {"cli": 1, "tests/data/pytest/tests/test_funcs1.py": 1, "test_func2": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.00017335500000004167, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/test_funcs1.py::test_func2", "location": ["tests/data/pytest/tests/test_funcs1.py", 4, "test_func2"], "keywords": {"cli": 1, "tests/data/pytest/tests/test_funcs1.py": 1, "test_func2": 1}, "outcome": "failed", "longrepr": {"reprcrash": {"path": "/Users/yabuki-ryosuke/src/github.com/launchableinc/cli/tests/data/pytest/tests/test_funcs1.py", "lineno": 6, "message": "assert 1 == False"}, "reprtraceback": {"reprentries": [{"type": "ReprEntry", "data": {"lines": [" def test_func2():", "> assert 1 == False", "E assert 1 == False"], "reprfuncargs": {"args": []}, "reprlocals": null, "reprfileloc": {"path": "tests/test_funcs1.py", "lineno": 6, "message": "AssertionError"}, "style": "long"}}], "extraline": null, "style": "long"}, "sections": [], "chain": [[{"reprentries": [{"type": "ReprEntry", "data": {"lines": [" def test_func2():", "> assert 1 == False", "E assert 1 == False"], "reprfuncargs": {"args": []}, "reprlocals": null, "reprfileloc": {"path": "tests/test_funcs1.py", "lineno": 6, "message": "AssertionError"}, "style": "long"}}], "extraline": null, "style": "long"}, {"path": "/Users/yabuki-ryosuke/src/github.com/launchableinc/cli/tests/data/pytest/tests/test_funcs1.py", "lineno": 6, "message": "assert 1 == False"}, null]]}, "when": "call", "user_properties": [], "sections": [], "duration": 0.001, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/test_funcs1.py::test_func2", "location": ["tests/data/pytest/tests/test_funcs1.py", 4, "test_func2"], "keywords": {"cli": 1, "tests/data/pytest/tests/test_funcs1.py": 1, "test_func2": 1}, "outcome": "passed", "longrepr": null, "when": "teardown", "user_properties": [], "sections": [], "duration": 0.0001447880000000623, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/test_funcs2.py::test_func3", "location": ["tests/data/pytest/tests/test_funcs2.py", 0, "test_func3"], "keywords": {"test_func3": 1, "cli": 1, "tests/data/pytest/tests/test_funcs2.py": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.00016672299999997975, "$report_type": "TestReport"} -{"nodeid": "tests/data/pytest/tests/test_funcs2.py::test_func3", "location": ["tests/data/pytest/tests/test_funcs2.py", 0, "test_func3"], "keywords": {"test_func3": 1, "cli": 1, "tests/data/pytest/tests/test_funcs2.py": 1}, "outcome": "passed", "longrepr": null, "when": "call", "user_properties": [], "sections": [], "duration": 0.001, "$report_type": "TestReport"} +{"nodeid": "tests/data/pytest/tests/test_funcs2.py::test_func3", "location": ["tests/data/pytest/tests/test_funcs2.py", 0, "test_func3"], "keywords": {"test_func3": 1, "cli": 1, "tests/data/pytest/tests/test_funcs2.py": 1}, "outcome": "passed", "longrepr": null, "when": "call", "user_properties": [["name", "order"], ["args", [2]], ["kwargs", {}]], "sections": [], "duration": 0.001, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/test_funcs2.py::test_func3", "location": ["tests/data/pytest/tests/test_funcs2.py", 0, "test_func3"], "keywords": {"test_func3": 1, "cli": 1, "tests/data/pytest/tests/test_funcs2.py": 1}, "outcome": "passed", "longrepr": null, "when": "teardown", "user_properties": [], "sections": [], "duration": 0.00023961299999997188, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/test_funcs2.py::test_func4", "location": ["tests/data/pytest/tests/test_funcs2.py", 4, "test_func4"], "keywords": {"test_func4": 1, "cli": 1, "tests/data/pytest/tests/test_funcs2.py": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.00015693899999991157, "$report_type": "TestReport"} {"nodeid": "tests/data/pytest/tests/test_funcs2.py::test_func4", "location": ["tests/data/pytest/tests/test_funcs2.py", 4, "test_func4"], "keywords": {"test_func4": 1, "cli": 1, "tests/data/pytest/tests/test_funcs2.py": 1}, "outcome": "passed", "longrepr": null, "when": "call", "user_properties": [], "sections": [], "duration": 0.001, "$report_type": "TestReport"}