diff --git a/launchable/test_runners/karma.py b/launchable/test_runners/karma.py new file mode 100644 index 000000000..ffd471208 --- /dev/null +++ b/launchable/test_runners/karma.py @@ -0,0 +1,153 @@ +# This runner only supports recording tests +# For subsetting, use 'ng' test runner instead +# It's possible to use 'karma' runner for recording, and 'ng' runner for subsetting, for the same test session +import json +from typing import Dict, Generator, List + +import click + +from ..commands.record.case_event import CaseEvent +from ..testpath import TestPath +from . import launchable + + +@click.argument('reports', required=True, nargs=-1) +@launchable.record.tests +def record_tests(client, reports): + client.parse_func = JSONReportParser(client).parse_func + + for r in reports: + client.report(r) + + client.run() + + +class JSONReportParser: + """ + Sample Karma report format: + { + "browsers": {...}, + "result": { + "24461741": [ + { + "fullName": "path/to/spec.ts should do something", + "description": "should do something", + "id": "spec0", + "log": [], + "skipped": false, + "disabled": false, + "pending": false, + "success": true, + "suite": [ + "path/to/spec.ts" + ], + "time": 92, + "executedExpectationsCount": 1, + "passedExpectations": [...], + "properties": null + } + ] + }, + "summary": {...} + } + """ + + def __init__(self, client): + self.client = client + + def parse_func(self, report_file: str) -> Generator[Dict, None, None]: # type: ignore + data: Dict + with open(report_file, 'r') as json_file: + try: + data = json.load(json_file) + except Exception: + click.echo( + click.style("Error: Failed to load Json report file: {}".format(report_file), fg='red'), err=True) + return + + if not self._validate_report_format(data): + click.echo( + "Error: {} does not appear to be valid Karma report format. " + "Make sure you are using karma-json-reporter or a compatible reporter.".format( + report_file), err=True) + return + + results = data.get("result", {}) + for browser_id, specs in results.items(): + if isinstance(specs, list): + for event in self._parse_specs(specs): + yield event + + def _validate_report_format(self, data: Dict) -> bool: + if not isinstance(data, dict): + return False + + if "result" not in data: + return False + + results = data.get("result", {}) + if not isinstance(results, dict): + return False + + for browser_id, specs in results.items(): + if not isinstance(specs, list): + return False + + for spec in specs: + if not isinstance(spec, dict): + return False + # Check for required fields + if "suite" not in spec or "time" not in spec: + return False + # Field suite should have at least one element (filename) + suite = spec.get("suite", []) + if not isinstance(suite, list) or len(suite) == 0: + return False + + return True + + def _parse_specs(self, specs: List[Dict]) -> List[Dict]: + events: List[Dict] = [] + + for spec in specs: + # TODO: + # In NextWorld, test filepaths are included in the suite tag + # But generally in a Karma test report, a suite tag can be any string + # For the time being let's get filepaths from the suite tag, + # until we find a standard way to include filepaths in the test reports + suite = spec.get("suite", []) + filename = suite[0] if suite else "" + + test_path: TestPath = [ + self.client.make_file_path_component(filename), + {"type": "testcase", "name": spec.get("fullName", spec.get("description", ""))} + ] + + duration_msec = spec.get("time", 0) + status = self._case_event_status_from_spec(spec) + stderr = self._parse_stderr(spec) + + events.append(CaseEvent.create( + test_path=test_path, + duration_secs=duration_msec / 1000 if duration_msec else 0, + status=status, + stderr=stderr + )) + + return events + + def _case_event_status_from_spec(self, spec: Dict) -> int: + if spec.get("skipped", False) or spec.get("disabled", False) or spec.get("pending", False): + return CaseEvent.TEST_SKIPPED + + if spec.get("success", False): + return CaseEvent.TEST_PASSED + else: + return CaseEvent.TEST_FAILED + + def _parse_stderr(self, spec: Dict) -> str: + log_messages = spec.get("log", []) + if not log_messages: + return "" + + return "\n".join(str(msg) for msg in log_messages if msg) diff --git a/launchable/test_runners/ng.py b/launchable/test_runners/ng.py new file mode 100644 index 000000000..09b4cc509 --- /dev/null +++ b/launchable/test_runners/ng.py @@ -0,0 +1,22 @@ +from . import launchable + + +@launchable.subset +def subset(client): + """ + Input format example: + src/app/feature/feature.component.spec.ts + src/app/service/service.service.spec.ts + + Output format: --include= format that can be passed to ng test + Example: + --include=src/app/feature/feature.component.spec.ts --include=src/app/service/service.service.spec.ts + """ + for t in client.stdin(): + path = t.strip() + if path: + client.test_path(path) + + client.formatter = lambda x: "--include={}".format(x[0]['name']) + client.separator = " " + client.run() diff --git a/tests/data/karma/record_test_result.json b/tests/data/karma/record_test_result.json new file mode 100644 index 000000000..d0972d26d --- /dev/null +++ b/tests/data/karma/record_test_result.json @@ -0,0 +1,189 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "foo/bar/zot.spec.ts" + }, + { + "type": "testcase", + "name": "foo/bar/zot.spec.ts should feed the monkey" + } + ], + "duration": 0.092, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "foo/bar/zot.spec.ts" + }, + { + "type": "testcase", + "name": "foo/bar/zot.spec.ts should fetch the toy" + } + ], + "duration": 0.027, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "foo/bar/zot.spec.ts" + }, + { + "type": "testcase", + "name": "foo/bar/zot.spec.ts should fetch the toyForRecord - record found" + } + ], + "duration": 0.028, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "foo/bar/zot.spec.ts" + }, + { + "type": "testcase", + "name": "foo/bar/zot.spec.ts should fetch the toyForRecord - record not found" + } + ], + "duration": 0.027, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "foo/bar/zot.spec.ts" + }, + { + "type": "testcase", + "name": "foo/bar/zot.spec.ts should fetch the toyForRecord - error in response" + } + ], + "duration": 0.033, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "foo/bar/zot.spec.ts" + }, + { + "type": "testcase", + "name": "foo/bar/zot.spec.ts should throw a ball" + } + ], + "duration": 0.026, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "foo/bar/zot.spec.ts" + }, + { + "type": "testcase", + "name": "foo/bar/zot.spec.ts should be nice to the zookeeper" + } + ], + "duration": 0.024, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "foo/bar/zot.spec.ts" + }, + { + "type": "testcase", + "name": "foo/bar/zot.spec.ts should like a banana or two" + } + ], + "duration": 0.024, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "foo/bar/zot.spec.ts" + }, + { + "type": "testcase", + "name": "foo/bar/zot.spec.ts should fall back to an apple if banana is not available" + } + ], + "duration": 0.025, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "foo/bar/zot.spec.ts" + }, + { + "type": "testcase", + "name": "foo/bar/zot.spec.ts should accept oranges if apple is not available" + } + ], + "duration": 0.032, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + } + ], + "testRunner": "karma", + "group": "", + "noBuild": false, + "flavors": [], + "testSuite": "" +} diff --git a/tests/data/karma/sample-report.json b/tests/data/karma/sample-report.json new file mode 100644 index 000000000..ada7cea12 --- /dev/null +++ b/tests/data/karma/sample-report.json @@ -0,0 +1,360 @@ +{ + "browsers": { + "123456": { + "id": "123456", + "fullName": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/131.0.0.0 Safari/537.36", + "name": "Chrome Headless 131.0.0.0 (Mac OS 10.15.7)", + "state": "DISCONNECTED", + "lastResult": { + "startTime": 1763588893304, + "total": 10, + "success": 10, + "failed": 0, + "skipped": 0, + "totalTime": 358, + "netTime": 338, + "error": true, + "disconnected": false + }, + "disconnectsCount": 0, + "noActivityTimeout": 30000, + "disconnectDelay": 2000 + } + }, + "result": { + "123456": [ + { + "fullName": "foo/bar/zot.spec.ts should feed the monkey", + "description": "should feed the monkey", + "id": "spec0", + "log": [], + "skipped": false, + "disabled": false, + "pending": false, + "success": true, + "suite": [ + "foo/bar/zot.spec.ts" + ], + "time": 92, + "executedExpectationsCount": 1, + "passedExpectations": [ + { + "matcherName": "toBeTruthy", + "message": "Passed.", + "stack": "", + "passed": true + } + ], + "properties": null + }, + { + "fullName": "foo/bar/zot.spec.ts should fetch the toy", + "description": "should fetch the toy", + "id": "spec1", + "log": [], + "skipped": false, + "disabled": false, + "pending": false, + "success": true, + "suite": [ + "foo/bar/zot.spec.ts" + ], + "time": 27, + "executedExpectationsCount": 7, + "passedExpectations": [ + { + "matcherName": "toHaveBeenCalled", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toHaveBeenCalled", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toHaveBeenCalled", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toHaveBeenCalled", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toHaveBeenCalled", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toHaveBeenCalledTimes", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toEqual", + "message": "Passed.", + "stack": "", + "passed": true + } + ], + "properties": null + }, + { + "fullName": "foo/bar/zot.spec.ts should fetch the toyForRecord - record found", + "description": "should fetch the toyForRecord - record found", + "id": "spec2", + "log": [], + "skipped": false, + "disabled": false, + "pending": false, + "success": true, + "suite": [ + "foo/bar/zot.spec.ts" + ], + "time": 28, + "executedExpectationsCount": 2, + "passedExpectations": [ + { + "matcherName": "toHaveBeenCalledOnceWith", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toEqual", + "message": "Passed.", + "stack": "", + "passed": true + } + ], + "properties": null + }, + { + "fullName": "foo/bar/zot.spec.ts should fetch the toyForRecord - record not found", + "description": "should fetch the toyForRecord - record not found", + "id": "spec3", + "log": [], + "skipped": false, + "disabled": false, + "pending": false, + "success": true, + "suite": [ + "foo/bar/zot.spec.ts" + ], + "time": 27, + "executedExpectationsCount": 2, + "passedExpectations": [ + { + "matcherName": "toHaveBeenCalledOnceWith", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toEqual", + "message": "Passed.", + "stack": "", + "passed": true + } + ], + "properties": null + }, + { + "fullName": "foo/bar/zot.spec.ts should fetch the toyForRecord - error in response", + "description": "should fetch the toyForRecord - error in response", + "id": "spec4", + "log": [], + "skipped": false, + "disabled": false, + "pending": false, + "success": true, + "suite": [ + "foo/bar/zot.spec.ts" + ], + "time": 33, + "executedExpectationsCount": 2, + "passedExpectations": [ + { + "matcherName": "toHaveBeenCalledOnceWith", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toEqual", + "message": "Passed.", + "stack": "", + "passed": true + } + ], + "properties": null + }, + { + "fullName": "foo/bar/zot.spec.ts should throw a ball", + "description": "should throw a ball", + "id": "spec5", + "log": [], + "skipped": false, + "disabled": false, + "pending": false, + "success": true, + "suite": [ + "foo/bar/zot.spec.ts" + ], + "time": 26, + "executedExpectationsCount": 2, + "passedExpectations": [ + { + "matcherName": "toEqual", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toHaveBeenCalledOnceWith", + "message": "Passed.", + "stack": "", + "passed": true + } + ], + "properties": null + }, + { + "fullName": "foo/bar/zot.spec.ts should be nice to the zookeeper", + "description": "should be nice to the zookeeper", + "id": "spec6", + "log": [], + "skipped": false, + "disabled": false, + "pending": false, + "success": true, + "suite": [ + "foo/bar/zot.spec.ts" + ], + "time": 24, + "executedExpectationsCount": 2, + "passedExpectations": [ + { + "matcherName": "toEqual", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toHaveBeenCalledOnceWith", + "message": "Passed.", + "stack": "", + "passed": true + } + ], + "properties": null + }, + { + "fullName": "foo/bar/zot.spec.ts should like a banana or two", + "description": "should like a banana or two", + "id": "spec7", + "log": [], + "skipped": false, + "disabled": false, + "pending": false, + "success": true, + "suite": [ + "foo/bar/zot.spec.ts" + ], + "time": 24, + "executedExpectationsCount": 2, + "passedExpectations": [ + { + "matcherName": "toEqual", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toHaveBeenCalledOnceWith", + "message": "Passed.", + "stack": "", + "passed": true + } + ], + "properties": null + }, + { + "fullName": "foo/bar/zot.spec.ts should fall back to an apple if banana is not available", + "description": "should fall back to an apple if banana is not available", + "id": "spec8", + "log": [], + "skipped": false, + "disabled": false, + "pending": false, + "success": true, + "suite": [ + "foo/bar/zot.spec.ts" + ], + "time": 25, + "executedExpectationsCount": 2, + "passedExpectations": [ + { + "matcherName": "toBeNull", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toHaveBeenCalledOnceWith", + "message": "Passed.", + "stack": "", + "passed": true + } + ], + "properties": null + }, + { + "fullName": "foo/bar/zot.spec.ts should accept oranges if apple is not available", + "description": "should accept oranges if apple is not available", + "id": "spec9", + "log": [], + "skipped": false, + "disabled": false, + "pending": false, + "success": true, + "suite": [ + "foo/bar/zot.spec.ts" + ], + "time": 32, + "executedExpectationsCount": 2, + "passedExpectations": [ + { + "matcherName": "toEqual", + "message": "Passed.", + "stack": "", + "passed": true + }, + { + "matcherName": "toHaveBeenCalledOnceWith", + "message": "Passed.", + "stack": "", + "passed": true + } + ], + "properties": null + } + ] + }, + "summary": { + "success": 10, + "failed": 0, + "skipped": 0, + "error": true, + "disconnected": false, + "exitCode": 1 + } +} diff --git a/tests/data/ng/subset_payload.json b/tests/data/ng/subset_payload.json new file mode 100644 index 000000000..d0b15b816 --- /dev/null +++ b/tests/data/ng/subset_payload.json @@ -0,0 +1,16 @@ +{ + "testPaths": [ + [ + { "type": "file", "name": "foo/bar/zot.spec.ts" } + ], + [ + { "type": "file", "name": "client-source/src/app/shared/other-test.spec.ts" } + ] + ], + "testRunner": "ng", + "goal": {"type": "subset-by-percentage", "percentage": 0.1}, + "ignoreNewTests": false, + "session": { "id": "16" }, + "getTestsFromGuess": false, + "getTestsFromPreviousSessions": false +} diff --git a/tests/test_runners/test_karma.py b/tests/test_runners/test_karma.py new file mode 100644 index 000000000..907904788 --- /dev/null +++ b/tests/test_runners/test_karma.py @@ -0,0 +1,18 @@ +import os +from unittest import mock + +import responses # type: ignore + +from tests.cli_test_case import CliTestCase + + +class KarmaTest(CliTestCase): + @responses.activate + @mock.patch.dict(os.environ, + {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_record_tests_json(self): + result = self.cli('record', 'tests', '--session', self.session, + 'karma', str(self.test_files_dir.joinpath("sample-report.json"))) + + self.assert_success(result) + self.assert_record_tests_payload('record_test_result.json') diff --git a/tests/test_runners/test_ng.py b/tests/test_runners/test_ng.py new file mode 100644 index 000000000..36c090c29 --- /dev/null +++ b/tests/test_runners/test_ng.py @@ -0,0 +1,22 @@ +import os +from unittest import mock + +import responses # type: ignore + +from launchable.utils.session import write_build +from tests.cli_test_case import CliTestCase + + +class NgTest(CliTestCase): + @responses.activate + @mock.patch.dict(os.environ, + {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset(self): + write_build(self.build_name) + + subset_input = """foo/bar/zot.spec.ts +client-source/src/app/shared/other-test.spec.ts +""" + result = self.cli('subset', '--target', '10%', 'ng', input=subset_input) + self.assert_success(result) + self.assert_subset_payload('subset_payload.json')