From 03f6780a544a6269a8d765c2193d1e2421188b29 Mon Sep 17 00:00:00 2001 From: Ryosuke Yabuki Date: Thu, 14 Nov 2024 23:47:25 +0900 Subject: [PATCH 1/6] init (but wip) --- launchable/test_runners/flutter.py | 230 +++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 launchable/test_runners/flutter.py diff --git a/launchable/test_runners/flutter.py b/launchable/test_runners/flutter.py new file mode 100644 index 000000000..8c90a33c7 --- /dev/null +++ b/launchable/test_runners/flutter.py @@ -0,0 +1,230 @@ +import json +from typing import Dict, Generator, List, Optional +import click + + +from launchable.commands.record.case_event import CaseEvent +from launchable.testpath import FilePathNormalizer +from . import launchable + + + +FLUTTER_FILE_EXT = "_test.dart" + +FLUTTER_TEST_RESULT_SUCCESS = "success" +FLUTTER_TEST_RESULT_FAILURE = "error" + + +class TestCase: + def __init__(self, id: int, name: str, is_skipped: bool = False): + self._id = id + self._name = name + self._is_skipped = is_skipped + self._status = None + self._stdout = "" + self._stderr = "" + self._duration_sec = 0 + + @property + def id(self): + return self._id + + @property + def name(self): + return self._name + + @property + def status(self) -> CaseEvent: + if self._is_skipped: + return CaseEvent.TEST_SKIPPED + elif self._status == FLUTTER_TEST_RESULT_SUCCESS: + return CaseEvent.TEST_PASSED + elif self._status == FLUTTER_TEST_RESULT_FAILURE: + return CaseEvent.TEST_FAILED + + # safe fallback + return CaseEvent.TEST_PASSED + + @status.setter + def status(self, status: str): + self._status = status + + @property + def duration(self) -> float: + return self._duration_sec + + @duration.setter + def duration(self, duration_sec: float): + self._duration_sec = duration_sec + + @property + def stdout(self) -> str: + return self._stdout + + @stdout.setter + def stdout(self, stdout: str): + self._stdout = stdout + + @property + def stderr(self) -> str: + return self._stderr + + @stderr.setter + def stderr(self, stderr: str): + self._stderr = stderr + + +class TestSuite: + def __init__(self, id: int, platform: str, path: str): + self._id = id + self._platform = platform + self._path = path + self._test_cases: List[TestCase] = [] + + def _get_test_case(self, id: int) -> Optional[TestCase]: + if id is None: + return + + for c in self._test_cases: + if c.id == id: + return c + + +class ReportParser: + def __init__(self, client): + self.client = client + self.file_path_normalizer = FilePathNormalizer(base_path=client.base_path, + no_base_path_inference=client.no_base_path_inference) + self._suites: List[TestSuite] = [] + + def _get_suite(self, suite_id: int) -> Optional[TestSuite]: + if suite_id is None: + return + + for s in self._suites: + if s._id == suite_id: + return s + + def _get_test(self, test_id: int) -> Optional[TestCase]: + if test_id is None: + return + + for s in self._suites: + for c in s._test_cases: + if c.id == test_id: + return c + + def _events(self) -> List[CaseEvent]: + events = [] + for s in self._suites: + for c in s._test_cases: + events.append(CaseEvent.create( + test_path=[{"type": "file", "name": s._path, }, {"type": "testcase", "name": c.name}], + duration_secs=c.duration, + status=c.status, + stdout=c.stdout, + stderr=c.stderr, + )) + + return events + + def parse_func(self, report_file: str) -> Generator[CaseEvent, None, None]: + with open(report_file, "r") as ndjson: + for j in ndjson: + if not j.strip(): + continue + data = json.loads(j) + self._parse_json(data) + + print("come on") + for event in self._events(): + print("event~~~",event) + yield event + + def _parse_json(self, data: Dict): + if not isinstance(data, Dict): + # Note: array sometimes comes in but won't use it + return + + data_type = data.get("type") + if data_type is None: + return + + elif data_type == "suite": + suite_data = data.get("suite") + if suite_data is None: + # it's might be invalid suite data + return + + self._suites.append( + TestSuite(suite_data.get("id"), suite_data.get("platform"), suite_data.get("path")) + ) + elif data_type == "testStart": + test_data = data.get("test") + if test_data is None: + # it's might be invalid test data + return + + if test_data.get("line") is None: + # Still set up test case + return + + suite_id = test_data.get("suiteID") + suite = self._get_suite(suite_id) + if suite_id is None or suite is None: + click.echo(click.style( + "Warning: Cannot find a parent test suite (id: {}). So won't send test result of {}".format( + suite_id, test_data.get("name")), fg='yellow'), err=True) + return + + id = test_data.get("id") + name = test_data.get("name") + metadata = test_data.get("metadata", {}) + is_skipped = metadata.get("skip", False) + suite._test_cases.append(TestCase(id, name, is_skipped)) + elif data_type == "testDone": + test_id = data.get("testID") + test = self._get_test(test_id) + + if test is None: + return + + test.status = data.get("result", "success") # safe fallback + duration_msec = data.get("duration", 0) + test.duration = duration_msec / 1000 # to sec + + elif data_type == "error": + test_id = data.get("testID") + test = self._get_test(test_id) + if test is None: + click.echo(click.style( + "Warning: Cannot find a test (id: {}). So we skip update stderr".format(test_id), fg='yellow'), + err=True) + return + test.stderr += ("\n" if test.stderr else "") + data.get("error", "") + + elif data_type == "print": + test_id = data.get("testID") + test = self._get_test(test_id) + if test is None: + click.echo(click.style( + "Warning: Cannot find a test (id: {}). So we skip update stdout".format(test_id), fg='yellow'), + err=True) + return + + test.stdout += ("\n" if test.stdout else "") + data.get("message", "") + else: + return + + +@click.argument('reports', required=True, nargs=-1) +@launchable.record.tests +def record_tests(client, reports): + for r in reports: + client.report(r) + client.parse_func = ReportParser(client).parse_func + client.run() + + +subset = launchable.CommonSubsetImpls(__name__).scan_files('*.dart') +split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() \ No newline at end of file From 324d1ac06b21506e89fd3ebe5dd42c0611ce6781 Mon Sep 17 00:00:00 2001 From: Ryosuke Yabuki Date: Fri, 15 Nov 2024 14:53:47 +0900 Subject: [PATCH 2/6] implement flutter plugin --- launchable/test_runners/flutter.py | 109 +++++++++++++++-------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/launchable/test_runners/flutter.py b/launchable/test_runners/flutter.py index 8c90a33c7..a1c68b339 100644 --- a/launchable/test_runners/flutter.py +++ b/launchable/test_runners/flutter.py @@ -1,14 +1,14 @@ import json +import pathlib from typing import Dict, Generator, List, Optional import click - +from pathlib import Path from launchable.commands.record.case_event import CaseEvent from launchable.testpath import FilePathNormalizer from . import launchable - FLUTTER_FILE_EXT = "_test.dart" FLUTTER_TEST_RESULT_SUCCESS = "success" @@ -17,13 +17,13 @@ class TestCase: def __init__(self, id: int, name: str, is_skipped: bool = False): - self._id = id - self._name = name - self._is_skipped = is_skipped - self._status = None - self._stdout = "" - self._stderr = "" - self._duration_sec = 0 + self._id: int = id + self._name: str = name + self._is_skipped: bool = is_skipped + self._status: str = "" + self._stdout: str = "" + self._stderr: str = "" + self._duration_sec: float = 0 @property def id(self): @@ -34,7 +34,7 @@ def name(self): return self._name @property - def status(self) -> CaseEvent: + def status(self) -> int: # status code see: case_event.py if self._is_skipped: return CaseEvent.TEST_SKIPPED elif self._status == FLUTTER_TEST_RESULT_SUCCESS: @@ -79,47 +79,39 @@ def __init__(self, id: int, platform: str, path: str): self._id = id self._platform = platform self._path = path - self._test_cases: List[TestCase] = [] + self._test_cases: Dict[int, TestCase] = {} def _get_test_case(self, id: int) -> Optional[TestCase]: - if id is None: - return - - for c in self._test_cases: - if c.id == id: - return c + return self._test_cases.get(id) class ReportParser: - def __init__(self, client): - self.client = client - self.file_path_normalizer = FilePathNormalizer(base_path=client.base_path, - no_base_path_inference=client.no_base_path_inference) - self._suites: List[TestSuite] = [] + def __init__(self, file_path_normalizer: FilePathNormalizer): + self.file_path_normalizer = file_path_normalizer + self._suites: Dict[int, TestSuite] = {} def _get_suite(self, suite_id: int) -> Optional[TestSuite]: - if suite_id is None: - return - - for s in self._suites: - if s._id == suite_id: - return s + return self._suites.get(suite_id) def _get_test(self, test_id: int) -> Optional[TestCase]: if test_id is None: - return + return None + + for s in self._suites.values(): + c = s._get_test_case(test_id) + if c is not None: + return c - for s in self._suites: - for c in s._test_cases: - if c.id == test_id: - return c + return None - def _events(self) -> List[CaseEvent]: + def _events(self) -> List: events = [] - for s in self._suites: - for c in s._test_cases: + for s in self._suites.values(): + for c in s._test_cases.values(): events.append(CaseEvent.create( - test_path=[{"type": "file", "name": s._path, }, {"type": "testcase", "name": c.name}], + test_path=[ + {"type": "file", "name": pathlib.Path(self.file_path_normalizer.relativize(s._path)).as_posix()}, + {"type": "testcase", "name": c.name}], duration_secs=c.duration, status=c.status, stdout=c.stdout, @@ -136,9 +128,7 @@ def parse_func(self, report_file: str) -> Generator[CaseEvent, None, None]: data = json.loads(j) self._parse_json(data) - print("come on") for event in self._events(): - print("event~~~",event) yield event def _parse_json(self, data: Dict): @@ -149,39 +139,48 @@ def _parse_json(self, data: Dict): data_type = data.get("type") if data_type is None: return - elif data_type == "suite": suite_data = data.get("suite") if suite_data is None: # it's might be invalid suite data return - self._suites.append( - TestSuite(suite_data.get("id"), suite_data.get("platform"), suite_data.get("path")) - ) + suite_id = suite_data.get("id") + if self._get_suite(suite_data.get("id")) is not None: + # already recorded + return + + self._suites[suite_id] = TestSuite(suite_id, suite_data.get("platform"), suite_data.get("path")) elif data_type == "testStart": test_data = data.get("test") + if test_data is None: # it's might be invalid test data return - if test_data.get("line") is None: - # Still set up test case + # Still set up test case, should skip return suite_id = test_data.get("suiteID") suite = self._get_suite(suite_id) + if suite_id is None or suite is None: click.echo(click.style( "Warning: Cannot find a parent test suite (id: {}). So won't send test result of {}".format( suite_id, test_data.get("name")), fg='yellow'), err=True) return - id = test_data.get("id") + test_id = test_data.get("id") + test = self._get_test(test_id) + if test is not None: + # already recorded + return + name = test_data.get("name") metadata = test_data.get("metadata", {}) is_skipped = metadata.get("skip", False) - suite._test_cases.append(TestCase(id, name, is_skipped)) + suite._test_cases[test_id] = TestCase(test_id, name, is_skipped) + return elif data_type == "testDone": test_id = data.get("testID") test = self._get_test(test_id) @@ -190,9 +189,9 @@ def _parse_json(self, data: Dict): return test.status = data.get("result", "success") # safe fallback - duration_msec = data.get("duration", 0) + duration_msec = data.get("time", 0) test.duration = duration_msec / 1000 # to sec - + return elif data_type == "error": test_id = data.get("testID") test = self._get_test(test_id) @@ -202,7 +201,7 @@ def _parse_json(self, data: Dict): err=True) return test.stderr += ("\n" if test.stderr else "") + data.get("error", "") - + return elif data_type == "print": test_id = data.get("testID") test = self._get_test(test_id) @@ -213,6 +212,7 @@ def _parse_json(self, data: Dict): return test.stdout += ("\n" if test.stdout else "") + data.get("message", "") + return else: return @@ -220,10 +220,11 @@ def _parse_json(self, data: Dict): @click.argument('reports', required=True, nargs=-1) @launchable.record.tests def record_tests(client, reports): - for r in reports: - client.report(r) - client.parse_func = ReportParser(client).parse_func - client.run() + file_path_normalizer = FilePathNormalizer(base_path=client.base_path, no_base_path_inference=client.no_base_path_inference) + client.parse_func = ReportParser(file_path_normalizer).parse_func + + launchable.CommonRecordTestImpls.load_report_files(client=client, source_roots=reports) + subset = launchable.CommonSubsetImpls(__name__).scan_files('*.dart') From b0aeba32c95eeee27db9e269ef33f70acf1a626b Mon Sep 17 00:00:00 2001 From: Ryosuke Yabuki Date: Fri, 15 Nov 2024 15:01:12 +0900 Subject: [PATCH 3/6] rm redundant return --- launchable/test_runners/flutter.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/launchable/test_runners/flutter.py b/launchable/test_runners/flutter.py index a1c68b339..61e75c864 100644 --- a/launchable/test_runners/flutter.py +++ b/launchable/test_runners/flutter.py @@ -180,7 +180,7 @@ def _parse_json(self, data: Dict): metadata = test_data.get("metadata", {}) is_skipped = metadata.get("skip", False) suite._test_cases[test_id] = TestCase(test_id, name, is_skipped) - return + elif data_type == "testDone": test_id = data.get("testID") test = self._get_test(test_id) @@ -191,7 +191,7 @@ def _parse_json(self, data: Dict): test.status = data.get("result", "success") # safe fallback duration_msec = data.get("time", 0) test.duration = duration_msec / 1000 # to sec - return + elif data_type == "error": test_id = data.get("testID") test = self._get_test(test_id) @@ -201,7 +201,7 @@ def _parse_json(self, data: Dict): err=True) return test.stderr += ("\n" if test.stderr else "") + data.get("error", "") - return + elif data_type == "print": test_id = data.get("testID") test = self._get_test(test_id) @@ -212,7 +212,7 @@ def _parse_json(self, data: Dict): return test.stdout += ("\n" if test.stdout else "") + data.get("message", "") - return + else: return @@ -226,6 +226,5 @@ def record_tests(client, reports): launchable.CommonRecordTestImpls.load_report_files(client=client, source_roots=reports) - subset = launchable.CommonSubsetImpls(__name__).scan_files('*.dart') split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() \ No newline at end of file From 4341b09f8c1f178fb276e65dfc696e1d14cf6380 Mon Sep 17 00:00:00 2001 From: Ryosuke Yabuki Date: Fri, 15 Nov 2024 15:06:04 +0900 Subject: [PATCH 4/6] fix lint --- launchable/test_runners/flutter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/launchable/test_runners/flutter.py b/launchable/test_runners/flutter.py index 61e75c864..1b7aa656b 100644 --- a/launchable/test_runners/flutter.py +++ b/launchable/test_runners/flutter.py @@ -1,13 +1,14 @@ import json import pathlib +from pathlib import Path from typing import Dict, Generator, List, Optional + import click -from pathlib import Path from launchable.commands.record.case_event import CaseEvent from launchable.testpath import FilePathNormalizer -from . import launchable +from . import launchable FLUTTER_FILE_EXT = "_test.dart" @@ -227,4 +228,4 @@ def record_tests(client, reports): subset = launchable.CommonSubsetImpls(__name__).scan_files('*.dart') -split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() \ No newline at end of file +split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() From bb08f4ab79896ab5e9fa83579d48b1fe3079683b Mon Sep 17 00:00:00 2001 From: Ryosuke Yabuki Date: Mon, 18 Nov 2024 10:38:33 +0900 Subject: [PATCH 5/6] Add test We don't support retry test and the case that doesn't execute `flutter pub get` before running tests, for now --- launchable/test_runners/flutter.py | 4 ++ tests/data/flutter/record_test_result.json | 61 ++++++++++++++++++++++ tests/data/flutter/report.json | 29 ++++++++++ 3 files changed, 94 insertions(+) create mode 100644 tests/data/flutter/record_test_result.json create mode 100644 tests/data/flutter/report.json diff --git a/launchable/test_runners/flutter.py b/launchable/test_runners/flutter.py index 1b7aa656b..1ea590813 100644 --- a/launchable/test_runners/flutter.py +++ b/launchable/test_runners/flutter.py @@ -122,6 +122,8 @@ def _events(self) -> List: return events def parse_func(self, report_file: str) -> Generator[CaseEvent, None, None]: + # TODO: Support cases that include information about `flutter pub get` + # see detail: https://github.com/launchableinc/examples/actions/runs/11884312142/job/33112309450 with open(report_file, "r") as ndjson: for j in ndjson: if not j.strip(): @@ -204,6 +206,8 @@ def _parse_json(self, data: Dict): test.stderr += ("\n" if test.stderr else "") + data.get("error", "") elif data_type == "print": + # It's difficult to identify the "Retry" case because Flutter reports it with the same test ID + # So we won't handle it at the moment. test_id = data.get("testID") test = self._get_test(test_id) if test is None: diff --git a/tests/data/flutter/record_test_result.json b/tests/data/flutter/record_test_result.json new file mode 100644 index 000000000..28cab3ebc --- /dev/null +++ b/tests/data/flutter/record_test_result.json @@ -0,0 +1,61 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "test/skip_widget_test.dart" + }, + { + "type": "testcase", + "name": "Counter increments smoke skip test" + } + ], + "duration": 1.562, + "status": 2, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "test/failure_widget_test.dart" + }, + { + "type": "testcase", + "name": "Counter increments smoke failure test" + } + ], + "duration": 2.046, + "status": 0, + "stdout": "\u2550\u2550\u2561 EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK \u255e\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\nThe following TestFailure was thrown running a test:\nExpected: exactly one matching candidate\n Actual: _TextWidgetFinder:\n Which: means none were found but one was expected\n\nWhen the exception was thrown, this was the stack:\n#4 main. (file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart:30:5)\n\n#5 testWidgets.. (package:flutter_test/src/widget_tester.dart:189:15)\n\n#6 TestWidgetsFlutterBinding._runTestBody (package:flutter_test/src/binding.dart:1032:5)\n\n\n(elided one frame from package:stack_trace)\n\nThis was caught by the test expectation on the following line:\n file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart line 30\nThe test description was:\n Counter increments smoke failure test\n\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550", + "stderr": "Test failed. See exception logs above.\nThe test description was: Counter increments smoke failure test", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "test/widget_test.dart" + }, + { + "type": "testcase", + "name": "Counter increments smoke test" + } + ], + "duration": 1.998, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + } + ], + "testRunner": "flutter", + "group": "", + "noBuild": false +} \ No newline at end of file diff --git a/tests/data/flutter/report.json b/tests/data/flutter/report.json new file mode 100644 index 000000000..67ef14e45 --- /dev/null +++ b/tests/data/flutter/report.json @@ -0,0 +1,29 @@ +{"protocolVersion":"0.1.1","runnerVersion":"1.25.7","pid":30535,"type":"start","time":0} +{"suite":{"id":0,"platform":"vm","path":"/Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/skip_widget_test.dart"},"type":"suite","time":0} +{"test":{"id":1,"name":"loading /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/skip_widget_test.dart","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":0} +{"suite":{"id":2,"platform":"vm","path":"/Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart"},"type":"suite","time":2} +{"test":{"id":3,"name":"loading /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart","suiteID":2,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":2} +{"suite":{"id":4,"platform":"vm","path":"/Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/widget_test.dart"},"type":"suite","time":3} +{"test":{"id":5,"name":"loading /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/widget_test.dart","suiteID":4,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":3} +{"count":3,"time":3,"type":"allSuites"} + +[{"event":"test.startedProcess","params":{"vmServiceUri":null,"observatoryUri":null}}] + +[{"event":"test.startedProcess","params":{"vmServiceUri":null,"observatoryUri":null}}] + +[{"event":"test.startedProcess","params":{"vmServiceUri":null,"observatoryUri":null}}] +{"testID":5,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1558} +{"group":{"id":6,"suiteID":4,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":null,"column":null,"url":null},"type":"group","time":1560} +{"test":{"id":7,"name":"Counter increments smoke test","suiteID":4,"groupIDs":[6],"metadata":{"skip":false,"skipReason":null},"line":171,"column":5,"url":"package:flutter_test/src/widget_tester.dart","root_line":14,"root_column":3,"root_url":"file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/widget_test.dart"},"type":"testStart","time":1560} +{"testID":3,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1561} +{"group":{"id":8,"suiteID":2,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":null,"column":null,"url":null},"type":"group","time":1561} +{"test":{"id":9,"name":"Counter increments smoke failure test","suiteID":2,"groupIDs":[8],"metadata":{"skip":false,"skipReason":null},"line":171,"column":5,"url":"package:flutter_test/src/widget_tester.dart","root_line":14,"root_column":3,"root_url":"file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart"},"type":"testStart","time":1561} +{"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1562} +{"group":{"id":10,"suiteID":0,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":null,"column":null,"url":null},"type":"group","time":1562} +{"test":{"id":11,"name":"Counter increments smoke skip test","suiteID":0,"groupIDs":[10],"metadata":{"skip":true,"skipReason":null},"line":171,"column":5,"url":"package:flutter_test/src/widget_tester.dart","root_line":14,"root_column":3,"root_url":"file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/skip_widget_test.dart"},"type":"testStart","time":1562} +{"testID":11,"result":"success","skipped":true,"hidden":false,"type":"testDone","time":1562} +{"testID":7,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":1998} +{"testID":9,"messageType":"print","message":"══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════\nThe following TestFailure was thrown running a test:\nExpected: exactly one matching candidate\n Actual: _TextWidgetFinder:\n Which: means none were found but one was expected\n\nWhen the exception was thrown, this was the stack:\n#4 main. (file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart:30:5)\n\n#5 testWidgets.. (package:flutter_test/src/widget_tester.dart:189:15)\n\n#6 TestWidgetsFlutterBinding._runTestBody (package:flutter_test/src/binding.dart:1032:5)\n\n\n(elided one frame from package:stack_trace)\n\nThis was caught by the test expectation on the following line:\n file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart line 30\nThe test description was:\n Counter increments smoke failure test\n════════════════════════════════════════════════════════════════════════════════════════════════════","type":"print","time":2042} +{"testID":9,"error":"Test failed. See exception logs above.\nThe test description was: Counter increments smoke failure test","stackTrace":"","isFailure":false,"type":"error","time":2044} +{"testID":9,"result":"error","skipped":false,"hidden":false,"type":"testDone","time":2046} +{"success":false,"type":"done","time":2052} From 9a0ab946286bfcf728d62f0224fe1d1b22a8040f Mon Sep 17 00:00:00 2001 From: Ryosuke Yabuki Date: Mon, 18 Nov 2024 11:33:26 +0900 Subject: [PATCH 6/6] error handling --- launchable/test_runners/flutter.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/launchable/test_runners/flutter.py b/launchable/test_runners/flutter.py index 1ea590813..1234b1c2f 100644 --- a/launchable/test_runners/flutter.py +++ b/launchable/test_runners/flutter.py @@ -124,12 +124,26 @@ def _events(self) -> List: def parse_func(self, report_file: str) -> Generator[CaseEvent, None, None]: # TODO: Support cases that include information about `flutter pub get` # see detail: https://github.com/launchableinc/examples/actions/runs/11884312142/job/33112309450 + if not pathlib.Path(report_file).exists(): + click.echo(click.style("Error: Report file not found: {}".format(report_file), fg='red'), err=True) + return + with open(report_file, "r") as ndjson: - for j in ndjson: - if not j.strip(): - continue - data = json.loads(j) - self._parse_json(data) + try: + for j in ndjson: + if not j.strip(): + continue + try: + data = json.loads(j) + self._parse_json(data) + except json.JSONDecodeError: + click.echo( + click.style("Error: Invalid JSON format: {}. Skip load this line".format(j), fg='yellow'), err=True) + continue + except Exception as e: + click.echo( + click.style("Error: Failed to parse the report file: {} : {}".format(report_file, e), fg='red'), err=True) + return for event in self._events(): yield event