From 692f41d930b453d30acf916e1022a2a27a9a06b0 Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Tue, 19 Aug 2025 15:49:02 +0900 Subject: [PATCH 01/46] feature: update CI config files for Launchable CLI v1 branch --- .github/workflows/e2e.yml | 2 +- .github/workflows/python-package.yml | 4 ++-- .github/workflows/python-publish.yml | 2 +- .tagpr | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index aa43036fa..a2852b692 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -2,7 +2,7 @@ name: e2e on: push: - branches: [main] + branches: [v1] workflow_dispatch: diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index aee24985c..f9357aead 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -6,12 +6,12 @@ name: Python package on: workflow_dispatch: push: - branches: [ main ] + branches: [ v1 ] paths-ignore: - 'WORKSPACE' - 'src/**' pull_request: - branches: [ main ] + branches: [ v1 ] paths-ignore: - 'WORKSPACE' - 'src/**' diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 17b2b2484..9ba369b50 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: push: branches: - - main + - v1 env: IMAGE_NAME: cloudbees/launchable diff --git a/.tagpr b/.tagpr index dc1d98b70..a508ee580 100644 --- a/.tagpr +++ b/.tagpr @@ -38,6 +38,6 @@ # [tagpr] vPrefix = true - releaseBranch = main + releaseBranch = v1 versionFile = - changelog = false From d33f1f8f8d17b77ef2efa499d4d5a0fae527f48f Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 5 Sep 2025 10:27:43 -0700 Subject: [PATCH 02/46] [AIENG-230] defined a switch to control the test selection behavior --- launchable/commands/subset.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index e0ec48e4a..595f9232d 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -207,6 +207,12 @@ help="get subset list from git managed files", is_flag=True, ) +@click.option( + "--use-case", + "use_case", + type=str, + hidden=True, # control PTS v2 test selection behavior. Non-committed, so hidden for now. +) @click.pass_context def subset( context: click.core.Context, @@ -235,6 +241,7 @@ def subset( prioritized_tests_mapping_file: Optional[TextIO] = None, test_suite: Optional[str] = None, is_get_tests_from_guess: bool = False, + use_case: Optional[str] = None, ): app = context.obj tracking_client = TrackingClient(Command.SUBSET, app=app) @@ -513,6 +520,9 @@ def get_payload( if prioritized_tests_mapping_file: payload['prioritizedTestsMapping'] = json.load(prioritized_tests_mapping_file) + if use_case: + payload["changesUnderTest"] = use_case + return payload def _collect_potential_test_files(self): From 4c867ec2a0c238e17d4c0b32af4aded8d055afe7 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 5 Sep 2025 10:37:14 -0700 Subject: [PATCH 03/46] [chore] report an error in subset call It looks like c41d9a6270d448168e722155b25ac04b7bdad5fa removed the `raise_for_status` check, without which error message from the server won't be reported, even as a warning. This breaks the subset call in case of the server failure, since `res.json()` looks as if it's returning an empty subset --- launchable/commands/subset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index e0ec48e4a..39cb0df6b 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -560,6 +560,7 @@ def request_subset(self) -> SubsetResult: # The status code 422 is returned when validation error of the test mapping file occurs. if res.status_code == 422: print_error_and_die("Error: {}".format(res.reason), Tracking.ErrorEvent.USER_ERROR) + res.raise_for_status() return SubsetResult.from_response(res.json()) except Exception as e: From 4e88a665a1ddc1cd3f09d60b245c18018a51af1e Mon Sep 17 00:00:00 2001 From: gayanW Date: Mon, 8 Sep 2025 16:15:45 +0900 Subject: [PATCH 04/46] Extend --link flag to support explicit kinds --- launchable/commands/helper.py | 2 + launchable/commands/record/build.py | 11 +-- launchable/commands/record/session.py | 30 +++----- launchable/commands/record/tests.py | 5 ++ launchable/utils/link.py | 83 +++++++++++++++++++- tests/commands/record/test_build.py | 47 ++++++++++++ tests/commands/record/test_session.py | 84 +++++++++++++++++++++ tests/commands/record/test_tests.py | 104 ++++++++++++++++++++++++++ tests/utils/test_link.py | 54 ++++++++++++- 9 files changed, 390 insertions(+), 30 deletions(-) diff --git a/launchable/commands/helper.py b/launchable/commands/helper.py index ebd83e887..6606d518c 100644 --- a/launchable/commands/helper.py +++ b/launchable/commands/helper.py @@ -124,6 +124,8 @@ def find_or_create_session( session_id = read_session(saved_build_name) if session_id: _check_observation_mode_status(session_id, is_observation, tracking_client=tracking_client, app=context.obj) + if links: + click.echo(click.style("WARNING: --link option is ignored since session already exists."), err=True) return session_id context.invoke( diff --git a/launchable/commands/record/build.py b/launchable/commands/record/build.py index 321aceeff..e5dd3aa13 100644 --- a/launchable/commands/record/build.py +++ b/launchable/commands/record/build.py @@ -7,7 +7,7 @@ import click from tabulate import tabulate -from launchable.utils.link import CIRCLECI_KEY, GITHUB_ACTIONS_KEY, JENKINS_URL_KEY, LinkKind, capture_link +from launchable.utils.link import CIRCLECI_KEY, GITHUB_ACTIONS_KEY, JENKINS_URL_KEY, capture_links from launchable.utils.tracking import Tracking, TrackingClient from ...utils import subprocess @@ -316,14 +316,7 @@ def synthesize_workspaces() -> List[Workspace]: def send(ws: List[Workspace]) -> Optional[str]: # figure out all the CI links to capture def compute_links(): - _links = capture_link(os.environ) - for k, v in links: - _links.append({ - "title": k, - "url": v, - "kind": LinkKind.CUSTOM_LINK.name, - }) - return _links + return capture_links(links, os.environ) try: payload = { diff --git a/launchable/commands/record/session.py b/launchable/commands/record/session.py index b5376e1a0..d751baa9d 100644 --- a/launchable/commands/record/session.py +++ b/launchable/commands/record/session.py @@ -8,7 +8,7 @@ import click from launchable.utils.click import DATETIME_WITH_TZ, validate_past_datetime -from launchable.utils.link import LinkKind, capture_link +from launchable.utils.link import capture_links from launchable.utils.tracking import Tracking, TrackingClient from ...utils.click import KEY_VALUE @@ -181,25 +181,17 @@ def session( flavor_dict = dict(flavor) - payload = { - "flavors": flavor_dict, - "isObservation": is_observation, - "noBuild": is_no_build, - "lineage": lineage, - "testSuite": test_suite, - "timestamp": timestamp.isoformat() if timestamp else None, - } - - _links = capture_link(os.environ) - for link in links: - _links.append({ - "title": link[0], - "url": link[1], - "kind": LinkKind.CUSTOM_LINK.name, - }) - payload["links"] = _links - try: + payload = { + "flavors": flavor_dict, + "isObservation": is_observation, + "noBuild": is_no_build, + "lineage": lineage, + "testSuite": test_suite, + "timestamp": timestamp.isoformat() if timestamp else None, + "links": capture_links(links, os.environ), + } + sub_path = "builds/{}/test_sessions".format(build_name) res = client.request("post", sub_path, payload=payload) diff --git a/launchable/commands/record/tests.py b/launchable/commands/record/tests.py index 64415692a..f86ba9d03 100644 --- a/launchable/commands/record/tests.py +++ b/launchable/commands/record/tests.py @@ -229,6 +229,11 @@ def tests( is_no_build = False + if session and links: + warn_and_exit_if_fail_fast_mode( + "WARNING: `--link` and `--session` are set together.\n--link option can't be used with existing sessions." + ) + try: if is_no_build: session_id = "builds/{}/test_sessions/{}".format(NO_BUILD_BUILD_NAME, NO_BUILD_TEST_SESSION_ID) diff --git a/launchable/utils/link.py b/launchable/utils/link.py index 093221551..9753e0d5c 100644 --- a/launchable/utils/link.py +++ b/launchable/utils/link.py @@ -1,5 +1,8 @@ +import re from enum import Enum -from typing import Dict, List, Mapping +from typing import Dict, List, Mapping, Sequence, Tuple + +import click JENKINS_URL_KEY = 'JENKINS_URL' JENKINS_BUILD_URL_KEY = 'BUILD_URL' @@ -18,6 +21,8 @@ CIRCLECI_BUILD_NUM_KEY = 'CIRCLE_BUILD_NUM' CIRCLECI_JOB_KEY = 'CIRCLE_JOB' +GITHUB_PR_REGEX = re.compile(r"^https://github\.com/[^/]+/[^/]+/pull/\d+$") + class LinkKind(Enum): @@ -71,3 +76,79 @@ def capture_link(env: Mapping[str, str]) -> List[Dict[str, str]]: }) return links + + +def capture_links_from_options(link_options: Sequence[Tuple[str, str]]) -> List[Dict[str, str]]: + """ + Validate user-provided link options, inferring the kind when not explicitly specified. + + Each link option is expected in the format "kind|title=url" or "title=url". + If the kind is not provided, it infers the kind based on the URL pattern. + + Returns: + A list of dictionaries, where each dictionary contains the validated title, URL, and kind for each link. + + Raises: + click.UsageError: If an invalid kind is provided or URL doesn't match with the specified kind. + """ + links = [] + for k, url in link_options: + # if k,v in format "kind|title=url" + if '|' in k: + kind, title = k.split('|', 1) + if kind not in valid_kinds(): + msg = ("Invalid kind '{}' passed to --link option.\n" + "Supported kinds are: {}".format(kind, valid_kinds())) + raise click.UsageError(click.style(msg, fg="red")) + + if not url_matches_kind(url, kind): + msg = ("Invalid url '{}' passed to --link option.\n" + "URL doesn't match with the specified kind: {}".format(url, kind)) + raise click.UsageError(click.style(msg, fg="red")) + + # if k,v in format "title=url" + else: + kind = infer_kind(url) + title = k + + links.append({ + "title": title, + "url": url, + "kind": kind, + }) + + return links + + +def capture_links(link_options: Sequence[Tuple[str, str]], env: Mapping[str, str]) -> List[Dict[str, str]]: + + links = capture_links_from_options(link_options) + + env_links = capture_link(env) + for env_link in env_links: + if not has_kind(links, env_link['kind']): + links.append(env_link) + + return links + + +def infer_kind(url: str) -> str: + if GITHUB_PR_REGEX.match(url): + return LinkKind.GITHUB_PULL_REQUEST.name + + return LinkKind.CUSTOM_LINK.name + + +def has_kind(input_links: List[Dict[str, str]], kind: str) -> bool: + return any(link for link in input_links if link['kind'] == kind) + + +def valid_kinds() -> List[str]: + return [kind.name for kind in LinkKind if kind != LinkKind.LINK_KIND_UNSPECIFIED] + + +def url_matches_kind(url: str, kind: str) -> bool: + if kind == LinkKind.GITHUB_PULL_REQUEST.name: + return bool(GITHUB_PR_REGEX.match(url)) + + return True diff --git a/tests/commands/record/test_build.py b/tests/commands/record/test_build.py index 5d6940867..6f6591146 100644 --- a/tests/commands/record/test_build.py +++ b/tests/commands/record/test_build.py @@ -308,3 +308,50 @@ def test_with_timestamp(self, mock_check_output): }, payload) self.assertEqual(read_build(), self.build_name) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_build_with_links(self): + # Invalid kind + result = self.cli( + "record", + "build", + "--no-commit-collection", + "--link", + "UNKNOWN_KIND|PR=https://github.com/launchableinc/cli/pull/1", + "--name", + self.build_name) + self.assertIn("Invalid kind 'UNKNOWN_KIND' passed to --link option", result.output) + + # Invalid URL + result = self.cli( + "record", + "build", + "--no-commit-collection", + "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/1/files", + "--name", + self.build_name) + self.assertIn("Invalid url 'https://github.com/launchableinc/cli/pull/1/files' passed to --link option", result.output) + + # Infer kind + result = self.cli( + "record", + "build", + "--no-commit-collection", + "--link", + "PR=https://github.com/launchableinc/cli/pull/1", + "--name", + self.build_name) + self.assert_success(result) + + # Explicit kind + result = self.cli( + "record", + "build", + "--no-commit-collection", + "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/1", + "--name", + self.build_name) + self.assert_success(result) diff --git a/tests/commands/record/test_session.py b/tests/commands/record/test_session.py index ddac44e15..b3206b802 100644 --- a/tests/commands/record/test_session.py +++ b/tests/commands/record/test_session.py @@ -5,6 +5,7 @@ import responses # type: ignore from launchable.utils.http_client import get_base_url +from launchable.utils.link import LinkKind from tests.cli_test_case import CliTestCase @@ -188,3 +189,86 @@ def test_run_session_with_timestamp(self): "testSuite": None, "timestamp": "2023-10-01T12:00:00+00:00", }, payload) + + @responses.activate + @mock.patch.dict(os.environ, { + "LAUNCHABLE_TOKEN": CliTestCase.launchable_token, + 'LANG': 'C.UTF-8', + "GITHUB_PULL_REQUEST_URL": "https://github.com/launchableinc/cli/pull/1", + }, clear=True) + def test_run_session_with_links(self): + # Endpoint to assert + endpoint = "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name) + + # Capture from environment + result = self.cli("record", "session", "--build", self.build_name) + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 0).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "", + "url": "https://github.com/launchableinc/cli/pull/1", + }], payload["links"]) + + # Priority check + result = self.cli("record", "session", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 1).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Infer kind + result = self.cli("record", "session", "--build", self.build_name, "--link", + "PR=https://github.com/launchableinc/cli/pull/2") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 2).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Explicit kind + result = self.cli("record", "session", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 3).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Multiple kinds + result = self.cli("record", "session", "--build", self.build_name, "--link", + "GITHUB_ACTIONS|=https://github.com/launchableinc/mothership/actions/runs/3747451612") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 4).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_ACTIONS.name, + "title": "", + "url": "https://github.com/launchableinc/mothership/actions/runs/3747451612", + }, + { + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "", + "url": "https://github.com/launchableinc/cli/pull/1", + }], payload["links"]) + + # Invalid kind + result = self.cli("record", "session", "--build", self.build_name, "--link", + "UNKNOWN_KIND|PR=https://github.com/launchableinc/cli/pull/2") + self.assertIn("Invalid kind 'UNKNOWN_KIND' passed to --link option", result.output) + + # Invalid URL + result = self.cli("record", "session", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2/files") + self.assertIn("Invalid url 'https://github.com/launchableinc/cli/pull/2/files' passed to --link option", result.output) diff --git a/tests/commands/record/test_tests.py b/tests/commands/record/test_tests.py index 1e8c122e0..57aa72e40 100644 --- a/tests/commands/record/test_tests.py +++ b/tests/commands/record/test_tests.py @@ -9,6 +9,7 @@ from launchable.commands.record.tests import INVALID_TIMESTAMP, parse_launchable_timeformat from launchable.utils.http_client import get_base_url +from launchable.utils.link import LinkKind from launchable.utils.no_build import NO_BUILD_BUILD_NAME, NO_BUILD_TEST_SESSION_ID from launchable.utils.session import write_build, write_session from tests.cli_test_case import CliTestCase @@ -107,3 +108,106 @@ def test_when_total_test_duration_zero(self): self.assert_success(result) self.assertIn("Total test duration is 0.", result.output) + + @responses.activate + @mock.patch.dict(os.environ, { + "LAUNCHABLE_TOKEN": CliTestCase.launchable_token, + "GITHUB_PULL_REQUEST_URL": "https://github.com/launchableinc/cli/pull/1", + }) + def test_with_links(self): + # Endpoint to assert + endpoint = "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name) + + # Capture from environment + write_build(self.build_name) + result = self.cli("record", "tests", "--build", self.build_name, 'maven', str(self.report_files_dir) + "**/reports/") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 0).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "", + "url": "https://github.com/launchableinc/cli/pull/1", + }], payload["links"]) + + # Priority check + write_build(self.build_name) + result = self.cli("record", "tests", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2", 'maven', + str(self.report_files_dir) + "**/reports/") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 1).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Infer kind + write_build(self.build_name) + result = self.cli("record", + "tests", + "--build", + self.build_name, + "--link", + "PR=https://github.com/launchableinc/cli/pull/2", + 'maven', + str(self.report_files_dir) + "**/reports/") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 2).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Explicit kind + write_build(self.build_name) + result = self.cli("record", "tests", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2", 'maven', + str(self.report_files_dir) + "**/reports/") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 3).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Invalid kind + write_build(self.build_name) + result = self.cli("record", + "tests", + "--build", + self.build_name, + "--link", + "UNKNOWN_KIND|PR=https://github.com/launchableinc/cli/pull/2", + 'maven', + str(self.report_files_dir) + "**/reports/") + self.assertIn("Invalid kind 'UNKNOWN_KIND' passed to --link option", result.output) + + # Invalid URL + write_build(self.build_name) + result = self.cli("record", "tests", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2/files", 'maven', + str(self.report_files_dir) + "**/reports/") + self.assertIn("Invalid url 'https://github.com/launchableinc/cli/pull/2/files' passed to --link option", result.output) + + # With --session flag + write_session(self.build_name, self.session) + result = self.cli("record", "tests", "--session", self.session, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2/files", 'maven', + str(self.report_files_dir) + "**/reports/") + self.assertIn("WARNING: `--link` and `--session` are set together", result.output) + self.assert_success(result) + + # Existing session + write_session(self.build_name, self.session) + result = self.cli("record", "tests", "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2/files", 'maven', + str(self.report_files_dir) + "**/reports/") + self.assertIn("WARNING: --link option is ignored since session already exists", result.output) + self.assert_success(result) diff --git a/tests/utils/test_link.py b/tests/utils/test_link.py index 3ee302b1c..b7527bb67 100644 --- a/tests/utils/test_link.py +++ b/tests/utils/test_link.py @@ -1,6 +1,8 @@ from unittest import TestCase -from launchable.utils.link import LinkKind, capture_link +import click + +from launchable.utils.link import LinkKind, capture_link, capture_links, capture_links_from_options class LinkTest(TestCase): @@ -44,3 +46,53 @@ def test_circleci(self): "title": "job (234)", "url": "https://circleci.com/build/123", }]) + + def test_capture_links_from_options(self): + # Invalid kind + link_options = [("INVALID_KIND|PR", "https://github.com/launchableinc/cli/pull/1")] + with self.assertRaises(click.UsageError): + capture_links_from_options(link_options) + + # Invalid URL + link_options = [("GITHUB_PULL_REQUEST|PR", "https://github.com/launchableinc/cli/pull/1/files")] + with self.assertRaises(click.UsageError): + capture_links_from_options(link_options) + + # Infer kind + link_options = [("PR", "https://github.com/launchableinc/cli/pull/1")] + self.assertEqual(capture_links_from_options(link_options), [{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/1", + }]) + + # Explicit kind + link_options = [("GITHUB_PULL_REQUEST|PR", "https://github.com/launchableinc/cli/pull/1")] + self.assertEqual(capture_links_from_options(link_options), [{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/1", + }]) + + def test_capture_links(self): + # Capture from environment + envs = { + "GITHUB_PULL_REQUEST_URL": "https://github.com/launchableinc/cli/pull/1" + } + link_options = [] + self.assertEqual(capture_links(link_options, envs), [{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "", + "url": "https://github.com/launchableinc/cli/pull/1", + }]) + + # Priority check + envs = { + "GITHUB_PULL_REQUEST_URL": "https://github.com/launchableinc/cli/pull/1" + } + link_options = [("GITHUB_PULL_REQUEST|PR", "https://github.com/launchableinc/cli/pull/2")] + self.assertEqual(capture_links(link_options, envs), [{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }]) From d6314f1ada975079f23eda69cb4eeee6c0721b33 Mon Sep 17 00:00:00 2001 From: gayanW Date: Mon, 8 Sep 2025 17:38:57 +0900 Subject: [PATCH 05/46] Refactor tests.commands.record.test_tests.TestsTest.test_with_links --- tests/commands/record/test_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/record/test_tests.py b/tests/commands/record/test_tests.py index 57aa72e40..10125e119 100644 --- a/tests/commands/record/test_tests.py +++ b/tests/commands/record/test_tests.py @@ -113,7 +113,7 @@ def test_when_total_test_duration_zero(self): @mock.patch.dict(os.environ, { "LAUNCHABLE_TOKEN": CliTestCase.launchable_token, "GITHUB_PULL_REQUEST_URL": "https://github.com/launchableinc/cli/pull/1", - }) + }, clear=True) def test_with_links(self): # Endpoint to assert endpoint = "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions".format( From 364ff7390d6755a82ae530ae16f32d802d0a734c Mon Sep 17 00:00:00 2001 From: Konboi Date: Tue, 9 Sep 2025 14:21:17 +0900 Subject: [PATCH 06/46] [chore] to display with pager --- launchable/commands/compare/subsets.py | 2 +- launchable/commands/inspect/subset.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/launchable/commands/compare/subsets.py b/launchable/commands/compare/subsets.py index d55047256..c6162f4e0 100644 --- a/launchable/commands/compare/subsets.py +++ b/launchable/commands/compare/subsets.py @@ -47,4 +47,4 @@ def subsets(file_before, file_after): (before, after, f"{diff:+}" if isinstance(diff, int) else diff, test) for before, after, diff, test in rows ] - click.echo(tabulate(tabular_data, headers=headers, tablefmt="github")) + click.echo_via_pager(tabulate(tabular_data, headers=headers, tablefmt="github")) diff --git a/launchable/commands/inspect/subset.py b/launchable/commands/inspect/subset.py index a13e98505..5ff0baeb1 100644 --- a/launchable/commands/inspect/subset.py +++ b/launchable/commands/inspect/subset.py @@ -65,7 +65,7 @@ def display(self): result._estimated_duration_sec, ] ) - click.echo(tabulate(rows, header, tablefmt="github", floatfmt=".2f")) + click.echo_via_pager(tabulate(rows, header, tablefmt="github", floatfmt=".2f")) class SubsetResultJSONDisplay(SubsetResultAbstractDisplay): From 31b5f27b50a555dcb42a36c0b2926fe0acd33fb6 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Wed, 10 Sep 2025 09:12:20 +0900 Subject: [PATCH 07/46] [AIENG-196] Add the new command for the early flake detection --- launchable/__main__.py | 2 + launchable/commands/retry/__init__.py | 13 +++ launchable/commands/retry/flake_detection.py | 86 ++++++++++++++++++++ launchable/test_runners/bazel.py | 2 + launchable/test_runners/file.py | 2 + launchable/test_runners/launchable.py | 30 +++++++ launchable/test_runners/raw.py | 2 + launchable/utils/commands.py | 1 + 8 files changed, 138 insertions(+) create mode 100644 launchable/commands/retry/__init__.py create mode 100644 launchable/commands/retry/flake_detection.py diff --git a/launchable/__main__.py b/launchable/__main__.py index ef81dfd9a..3128cde5c 100644 --- a/launchable/__main__.py +++ b/launchable/__main__.py @@ -12,6 +12,7 @@ from .commands.compare import compare from .commands.inspect import inspect from .commands.record import record +from .commands.retry import retry from .commands.split_subset import split_subset from .commands.stats import stats from .commands.subset import subset @@ -91,6 +92,7 @@ def main(ctx, log_level, plugin_dir, dry_run, skip_cert_verification): main.add_command(inspect) main.add_command(stats) main.add_command(compare) +main.add_command(retry) if __name__ == '__main__': main() diff --git a/launchable/commands/retry/__init__.py b/launchable/commands/retry/__init__.py new file mode 100644 index 000000000..7b11e6e7f --- /dev/null +++ b/launchable/commands/retry/__init__.py @@ -0,0 +1,13 @@ +import click + +from launchable.utils.click import GroupWithAlias + +from .flake_detection import flake_detection + + +@click.group(cls=GroupWithAlias) +def retry(): + pass + + +retry.add_command(flake_detection, 'flake-detection') diff --git a/launchable/commands/retry/flake_detection.py b/launchable/commands/retry/flake_detection.py new file mode 100644 index 000000000..8b225913c --- /dev/null +++ b/launchable/commands/retry/flake_detection.py @@ -0,0 +1,86 @@ +import os +import sys + +import click + +from launchable.app import Application +from launchable.commands.helper import find_or_create_session +from launchable.commands.test_path_writer import TestPathWriter +from launchable.utils.click import ignorable_error +from launchable.utils.env_keys import REPORT_ERROR_KEY +from launchable.utils.launchable_client import LaunchableClient +from launchable.utils.tracking import Tracking, TrackingClient + +from ...utils.commands import Command + + +@click.group(help="Early flake detection") +@click.option( + '--session', + 'session', + help='In the format builds//test_sessions/', + type=str, + required=True +) +@click.option( + '--confidence', + help='Confidence level for flake detection', + type=click.Choice(['low', 'medium', 'high'], case_sensitive=False), + required=True, +) +@click.pass_context +def flake_detection(ctx, confidence, session): + tracking_client = TrackingClient(Command.FLAKE_DETECTION, app=ctx.obj) + client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client, test_runner=ctx.invoked_subcommand) + session_id = None + try: + session_id = find_or_create_session( + context=ctx, + session=session, + build_name=None, + tracking_client=tracking_client + ) + except click.UsageError as e: + click.echo(click.style(str(e), fg="red"), err=True) + sys.exit(1) + except Exception as e: + tracking_client.send_error_event( + event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, + stack_trace=str(e), + ) + if os.getenv(REPORT_ERROR_KEY): + raise e + else: + click.echo(ignorable_error(e), err=True) + if session_id is None: + return + + class FlakeDetection(TestPathWriter): + def __init__(self, app: Application): + super(FlakeDetection, self).__init__(app=app) + + def run(self): + test_paths = [] + try: + res = client.request( + "get", + "retry/flake-detection", + params={ + "confidence": confidence.upper(), + "session-id": os.path.basename(session_id), + "test-runner": ctx.invoked_subcommand}) + res.raise_for_status() + test_paths = res.json().get("testPaths", []) + if test_paths: + self.print(test_paths) + except Exception as e: + tracking_client.send_error_event( + event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, + stack_trace=str(e), + ) + if os.getenv(REPORT_ERROR_KEY): + raise e + else: + click.echo(ignorable_error(e), err=True) + + ctx.obj = FlakeDetection(app=ctx.obj) diff --git a/launchable/test_runners/bazel.py b/launchable/test_runners/bazel.py index 1904a926b..a76257d75 100644 --- a/launchable/test_runners/bazel.py +++ b/launchable/test_runners/bazel.py @@ -33,6 +33,8 @@ def subset(client): split_subset = launchable.CommonSplitSubsetImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).split_subset() +launchable.CommonFlakeDetectionImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).flake_detection() + @click.argument('workspace', required=True) @click.option('--build-event-json', 'build_event_json_files', help="set file path generated by --build_event_json_file", diff --git a/launchable/test_runners/file.py b/launchable/test_runners/file.py index 8c36746fc..dcfb4e2c5 100644 --- a/launchable/test_runners/file.py +++ b/launchable/test_runners/file.py @@ -52,3 +52,5 @@ def find_filename(): split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() + +launchable.CommonFlakeDetectionImpls(__name__).flake_detection() diff --git a/launchable/test_runners/launchable.py b/launchable/test_runners/launchable.py index 92119326c..8635c2936 100644 --- a/launchable/test_runners/launchable.py +++ b/launchable/test_runners/launchable.py @@ -6,8 +6,10 @@ import click from launchable.commands.record.tests import tests as record_tests_cmd +from launchable.commands.retry.flake_detection import flake_detection as flake_detection_cmd from launchable.commands.split_subset import split_subset as split_subset_cmd from launchable.commands.subset import subset as subset_cmd +from launchable.testpath import unparse_test_path def cmdname(m): @@ -43,6 +45,10 @@ def subset(f): record.tests = lambda f: wrap(f, record_tests_cmd) +def flake_detection(f): + return wrap(f, flake_detection_cmd) + + def split_subset(f): return wrap(f, split_subset_cmd) @@ -165,3 +171,27 @@ def split_subset(client): client.run() return wrap(split_subset, split_subset_cmd, self.cmdname) + + +class CommonFlakeDetectionImpls: + def __init__( + self, + module_name, + formatter=unparse_test_path, + seperator="\n", + ): + self.cmdname = cmdname(module_name) + self._formatter = formatter + self._separator = seperator + + def flake_detection(self): + def flake_detection(client): + if self._formatter: + client.formatter = self._formatter + + if self._separator: + client.separator = self._separator + + client.run() + + return wrap(flake_detection, flake_detection_cmd, self.cmdname) diff --git a/launchable/test_runners/raw.py b/launchable/test_runners/raw.py index cf52d0245..4ed980769 100644 --- a/launchable/test_runners/raw.py +++ b/launchable/test_runners/raw.py @@ -47,6 +47,8 @@ def subset(client, test_path_file): split_subset = launchable.CommonSplitSubsetImpls(__name__, formatter=unparse_test_path, seperator='\n').split_subset() +launchable.CommonFlakeDetectionImpls(__name__).flake_detection() + @click.argument('test_result_files', required=True, type=click.Path(exists=True), nargs=-1) @launchable.record.tests diff --git a/launchable/utils/commands.py b/launchable/utils/commands.py index 6a8e357b6..bd65a52ba 100644 --- a/launchable/utils/commands.py +++ b/launchable/utils/commands.py @@ -8,6 +8,7 @@ class Command(Enum): RECORD_SESSION = 'RECORD_SESSION' SUBSET = 'SUBSET' COMMIT = 'COMMIT' + FLAKE_DETECTION = 'FLAKE_DETECTION' def display_name(self): return self.value.lower().replace('_', ' ') From b55e8ed044f437399b9a2f8c5f1485f4e136164b Mon Sep 17 00:00:00 2001 From: gayanW Date: Wed, 10 Sep 2025 13:52:48 +0900 Subject: [PATCH 08/46] Prefix _ to private method names in utils/link.py --- launchable/utils/link.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/launchable/utils/link.py b/launchable/utils/link.py index 9753e0d5c..fb631242c 100644 --- a/launchable/utils/link.py +++ b/launchable/utils/link.py @@ -96,19 +96,19 @@ def capture_links_from_options(link_options: Sequence[Tuple[str, str]]) -> List[ # if k,v in format "kind|title=url" if '|' in k: kind, title = k.split('|', 1) - if kind not in valid_kinds(): + if kind not in _valid_kinds(): msg = ("Invalid kind '{}' passed to --link option.\n" - "Supported kinds are: {}".format(kind, valid_kinds())) + "Supported kinds are: {}".format(kind, _valid_kinds())) raise click.UsageError(click.style(msg, fg="red")) - if not url_matches_kind(url, kind): + if not _url_matches_kind(url, kind): msg = ("Invalid url '{}' passed to --link option.\n" "URL doesn't match with the specified kind: {}".format(url, kind)) raise click.UsageError(click.style(msg, fg="red")) # if k,v in format "title=url" else: - kind = infer_kind(url) + kind = _infer_kind(url) title = k links.append({ @@ -126,28 +126,28 @@ def capture_links(link_options: Sequence[Tuple[str, str]], env: Mapping[str, str env_links = capture_link(env) for env_link in env_links: - if not has_kind(links, env_link['kind']): + if not _has_kind(links, env_link['kind']): links.append(env_link) return links -def infer_kind(url: str) -> str: +def _infer_kind(url: str) -> str: if GITHUB_PR_REGEX.match(url): return LinkKind.GITHUB_PULL_REQUEST.name return LinkKind.CUSTOM_LINK.name -def has_kind(input_links: List[Dict[str, str]], kind: str) -> bool: +def _has_kind(input_links: List[Dict[str, str]], kind: str) -> bool: return any(link for link in input_links if link['kind'] == kind) -def valid_kinds() -> List[str]: +def _valid_kinds() -> List[str]: return [kind.name for kind in LinkKind if kind != LinkKind.LINK_KIND_UNSPECIFIED] -def url_matches_kind(url: str, kind: str) -> bool: +def _url_matches_kind(url: str, kind: str) -> bool: if kind == LinkKind.GITHUB_PULL_REQUEST.name: return bool(GITHUB_PR_REGEX.match(url)) From 56ee7f5b2fbe58282915ae03eda0ffb04c0de98d Mon Sep 17 00:00:00 2001 From: gayanW Date: Wed, 10 Sep 2025 14:37:42 +0900 Subject: [PATCH 09/46] Strip values pass to --link option --- launchable/utils/link.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/launchable/utils/link.py b/launchable/utils/link.py index fb631242c..a017a1513 100644 --- a/launchable/utils/link.py +++ b/launchable/utils/link.py @@ -93,9 +93,11 @@ def capture_links_from_options(link_options: Sequence[Tuple[str, str]]) -> List[ """ links = [] for k, url in link_options: + url = url.strip() + # if k,v in format "kind|title=url" if '|' in k: - kind, title = k.split('|', 1) + kind, title = (part.strip() for part in k.split('|', 1)) if kind not in _valid_kinds(): msg = ("Invalid kind '{}' passed to --link option.\n" "Supported kinds are: {}".format(kind, _valid_kinds())) @@ -109,7 +111,7 @@ def capture_links_from_options(link_options: Sequence[Tuple[str, str]]) -> List[ # if k,v in format "title=url" else: kind = _infer_kind(url) - title = k + title = k.strip() links.append({ "title": title, From 072ca6c6dfe733c42497143fe7bd18799fe4719b Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Wed, 10 Sep 2025 14:57:18 +0900 Subject: [PATCH 10/46] Add tests --- tests/commands/test_flake_detection.py | 83 ++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/commands/test_flake_detection.py diff --git a/tests/commands/test_flake_detection.py b/tests/commands/test_flake_detection.py new file mode 100644 index 000000000..4589b8c1f --- /dev/null +++ b/tests/commands/test_flake_detection.py @@ -0,0 +1,83 @@ +import os +from unittest import mock + +import responses # type: ignore + +from launchable.utils.http_client import get_base_url +from tests.cli_test_case import CliTestCase + + +class FlakeDetectionTest(CliTestCase): + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_success(self): + mock_json_response = { + "testPaths": [ + [{"type": "file", "name": "test_flaky_1.py"}], + [{"type": "file", "name": "test_flaky_2.py"}], + ] + } + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + json=mock_json_response, + status=200, + ) + result = self.cli( + "retry", + "flake-detection", + "--session", + self.session, + "--confidence", + "high", + "file", + mix_stderr=False, + ) + self.assert_success(result) + self.assertIn("test_flaky_1.py", result.stdout) + self.assertIn("test_flaky_2.py", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_no_flakes(self): + mock_json_response = {"testPaths": []} + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + json=mock_json_response, + status=200, + ) + result = self.cli( + "retry", + "flake-detection", + "--session", + self.session, + "--confidence", + "low", + "file", + mix_stderr=False, + ) + self.assert_success(result) + self.assertEqual(result.stdout, "") + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_api_error(self): + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + status=500, + ) + result = self.cli( + "retry", + "flake-detection", + "--session", + self.session, + "--confidence", + "medium", + "file", + mix_stderr=False, + ) + self.assert_exit_code(result, 0) + self.assertIn("Error", result.stderr) + self.assertEqual(result.stdout, "") From b7cc590425c57e86a54537fb31569d4cb40f5475 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 10 Sep 2025 21:50:55 +0000 Subject: [PATCH 11/46] [tagpr] prepare for the next release From 16afcef07ac921abb8d2edc18bfcfa48d5c401c4 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 15 Sep 2025 10:24:28 +0900 Subject: [PATCH 12/46] [AIENG-230] better error diagnostics --- launchable/commands/subset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 595f9232d..3e05e0e71 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -210,7 +210,7 @@ @click.option( "--use-case", "use_case", - type=str, + type=click.Choice(["one-commit", "feature-branch", "recurring"]), hidden=True, # control PTS v2 test selection behavior. Non-committed, so hidden for now. ) @click.pass_context From e93eed7c54af2d488ec1d9b945f756f148685182 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Tue, 16 Sep 2025 13:26:52 +0900 Subject: [PATCH 13/46] Fix CI on the v1 branch --- tests/commands/test_api_error.py | 10 +++++++++- tests/utils/test_http_client.py | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/commands/test_api_error.py b/tests/commands/test_api_error.py index 7c8cd16fd..8a9249a35 100644 --- a/tests/commands/test_api_error.py +++ b/tests/commands/test_api_error.py @@ -3,6 +3,7 @@ import platform import tempfile import threading +import time from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path from unittest import mock @@ -464,4 +465,11 @@ def test_all_workflow_when_server_down(self): def assert_tracking_count(self, tracking, count: int): # Prior to 3.6, `Response` object can't be obtained. if compare_version([int(x) for x in platform.python_version().split('.')], [3, 7]) >= 0: - assert tracking.call_count == count + # Sometimes, `tracking.call_count` is not updated immediately. So, wait a moment until it is updated. + attempt = 0 + while tracking.call_count < count: + time.sleep(0.1) + attempt += 1 + if attempt > 10: + break + self.assertEqual(tracking.call_count, count) diff --git a/tests/utils/test_http_client.py b/tests/utils/test_http_client.py index a565e2286..9fb4f8704 100644 --- a/tests/utils/test_http_client.py +++ b/tests/utils/test_http_client.py @@ -51,8 +51,8 @@ def test_reason(self): # use new session to disable retry cli = _HttpClient(session=Session()) - # /error is an actual endpoint that exists on our service to test the behavior - res = cli.request("GET", "intake/error") + # /raise_error is an actual endpoint that exists on our service to test the behavior + res = cli.request("GET", "intake/raise_error") self.assertEqual(res.status_code, 500) self.assertEqual(res.reason, "Welp") From 68eb8e157a3c616ef576c8bbbbe88b87e9b86edf Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Tue, 16 Sep 2025 14:30:33 +0900 Subject: [PATCH 14/46] Remove a useless test --- tests/utils/test_http_client.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/utils/test_http_client.py b/tests/utils/test_http_client.py index 9fb4f8704..aadb886ce 100644 --- a/tests/utils/test_http_client.py +++ b/tests/utils/test_http_client.py @@ -2,8 +2,6 @@ import platform from unittest import TestCase, mock -from requests import Session - from launchable.utils.http_client import _HttpClient from launchable.version import __version__ @@ -45,19 +43,3 @@ def test_header(self): "dummy", ), }) - - def test_reason(self): - '''make sure we correctly propagate error message from the server''' - - # use new session to disable retry - cli = _HttpClient(session=Session()) - # /raise_error is an actual endpoint that exists on our service to test the behavior - res = cli.request("GET", "intake/raise_error") - self.assertEqual(res.status_code, 500) - self.assertEqual(res.reason, "Welp") - - try: - res.raise_for_status() - self.fail("should have raised") - except Exception as e: - self.assertIn("Welp", str(e)) From 135c3ad99bb1358613678c95cfec677c1284d999 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Sep 2025 05:37:51 +0000 Subject: [PATCH 15/46] [tagpr] prepare for the next release From 5a1c2876c667c919b3f401f41934ca196ea9444d Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Tue, 16 Sep 2025 13:12:23 +0900 Subject: [PATCH 16/46] Rename `retry flake-detection` to the `detect-flake` command --- launchable/__main__.py | 4 +- .../flake_detection.py => detect_flakes.py} | 16 +++--- launchable/commands/retry/__init__.py | 13 ----- launchable/test_runners/bazel.py | 2 +- launchable/test_runners/file.py | 2 +- launchable/test_runners/launchable.py | 10 ++-- launchable/test_runners/raw.py | 2 +- launchable/test_runners/rspec.py | 1 + launchable/utils/commands.py | 2 +- ...ake_detection.py => test_detect_flakes.py} | 49 ++++++++++++++----- 10 files changed, 57 insertions(+), 44 deletions(-) rename launchable/commands/{retry/flake_detection.py => detect_flakes.py} (87%) delete mode 100644 launchable/commands/retry/__init__.py rename tests/commands/{test_flake_detection.py => test_detect_flakes.py} (63%) diff --git a/launchable/__main__.py b/launchable/__main__.py index 3128cde5c..34d8b5fa8 100644 --- a/launchable/__main__.py +++ b/launchable/__main__.py @@ -10,9 +10,9 @@ from launchable.app import Application from .commands.compare import compare +from .commands.detect_flakes import detect_flakes from .commands.inspect import inspect from .commands.record import record -from .commands.retry import retry from .commands.split_subset import split_subset from .commands.stats import stats from .commands.subset import subset @@ -92,7 +92,7 @@ def main(ctx, log_level, plugin_dir, dry_run, skip_cert_verification): main.add_command(inspect) main.add_command(stats) main.add_command(compare) -main.add_command(retry) +main.add_command(detect_flakes, "detect-flakes") if __name__ == '__main__': main() diff --git a/launchable/commands/retry/flake_detection.py b/launchable/commands/detect_flakes.py similarity index 87% rename from launchable/commands/retry/flake_detection.py rename to launchable/commands/detect_flakes.py index 8b225913c..3fcad9321 100644 --- a/launchable/commands/retry/flake_detection.py +++ b/launchable/commands/detect_flakes.py @@ -11,7 +11,7 @@ from launchable.utils.launchable_client import LaunchableClient from launchable.utils.tracking import Tracking, TrackingClient -from ...utils.commands import Command +from ..utils.commands import Command @click.group(help="Early flake detection") @@ -23,14 +23,16 @@ required=True ) @click.option( - '--confidence', - help='Confidence level for flake detection', + '--retry-threshold', + 'retry_threshold', + help='Throughness of how "flake" is detected', type=click.Choice(['low', 'medium', 'high'], case_sensitive=False), + default='medium', required=True, ) @click.pass_context -def flake_detection(ctx, confidence, session): - tracking_client = TrackingClient(Command.FLAKE_DETECTION, app=ctx.obj) +def detect_flakes(ctx, retry_threshold, session): + tracking_client = TrackingClient(Command.DETECT_FLAKE, app=ctx.obj) client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client, test_runner=ctx.invoked_subcommand) session_id = None try: @@ -64,9 +66,9 @@ def run(self): try: res = client.request( "get", - "retry/flake-detection", + "detect-flake", params={ - "confidence": confidence.upper(), + "confidence": retry_threshold.upper(), "session-id": os.path.basename(session_id), "test-runner": ctx.invoked_subcommand}) res.raise_for_status() diff --git a/launchable/commands/retry/__init__.py b/launchable/commands/retry/__init__.py deleted file mode 100644 index 7b11e6e7f..000000000 --- a/launchable/commands/retry/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -import click - -from launchable.utils.click import GroupWithAlias - -from .flake_detection import flake_detection - - -@click.group(cls=GroupWithAlias) -def retry(): - pass - - -retry.add_command(flake_detection, 'flake-detection') diff --git a/launchable/test_runners/bazel.py b/launchable/test_runners/bazel.py index a76257d75..372716dd1 100644 --- a/launchable/test_runners/bazel.py +++ b/launchable/test_runners/bazel.py @@ -33,7 +33,7 @@ def subset(client): split_subset = launchable.CommonSplitSubsetImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).split_subset() -launchable.CommonFlakeDetectionImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).flake_detection() +launchable.CommonFlakeDetectionImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).detect_flakes() @click.argument('workspace', required=True) diff --git a/launchable/test_runners/file.py b/launchable/test_runners/file.py index dcfb4e2c5..3b9a11549 100644 --- a/launchable/test_runners/file.py +++ b/launchable/test_runners/file.py @@ -53,4 +53,4 @@ def find_filename(): split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() -launchable.CommonFlakeDetectionImpls(__name__).flake_detection() +launchable.CommonFlakeDetectionImpls(__name__).detect_flakes() diff --git a/launchable/test_runners/launchable.py b/launchable/test_runners/launchable.py index 8635c2936..33d60b71e 100644 --- a/launchable/test_runners/launchable.py +++ b/launchable/test_runners/launchable.py @@ -5,8 +5,8 @@ import click +from launchable.commands.detect_flakes import detect_flakes as detect_flakes_cmd from launchable.commands.record.tests import tests as record_tests_cmd -from launchable.commands.retry.flake_detection import flake_detection as flake_detection_cmd from launchable.commands.split_subset import split_subset as split_subset_cmd from launchable.commands.subset import subset as subset_cmd from launchable.testpath import unparse_test_path @@ -46,7 +46,7 @@ def subset(f): def flake_detection(f): - return wrap(f, flake_detection_cmd) + return wrap(f, detect_flakes_cmd) def split_subset(f): @@ -184,8 +184,8 @@ def __init__( self._formatter = formatter self._separator = seperator - def flake_detection(self): - def flake_detection(client): + def detect_flakes(self): + def detect_flakes(client): if self._formatter: client.formatter = self._formatter @@ -194,4 +194,4 @@ def flake_detection(client): client.run() - return wrap(flake_detection, flake_detection_cmd, self.cmdname) + return wrap(detect_flakes, detect_flakes_cmd, self.cmdname) diff --git a/launchable/test_runners/raw.py b/launchable/test_runners/raw.py index 4ed980769..565946b17 100644 --- a/launchable/test_runners/raw.py +++ b/launchable/test_runners/raw.py @@ -47,7 +47,7 @@ def subset(client, test_path_file): split_subset = launchable.CommonSplitSubsetImpls(__name__, formatter=unparse_test_path, seperator='\n').split_subset() -launchable.CommonFlakeDetectionImpls(__name__).flake_detection() +launchable.CommonFlakeDetectionImpls(__name__).detect_flakes() @click.argument('test_result_files', required=True, type=click.Path(exists=True), nargs=-1) diff --git a/launchable/test_runners/rspec.py b/launchable/test_runners/rspec.py index 15ffcc155..312137c80 100644 --- a/launchable/test_runners/rspec.py +++ b/launchable/test_runners/rspec.py @@ -3,3 +3,4 @@ subset = launchable.CommonSubsetImpls(__name__).scan_files('*_spec.rb') split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() record_tests = launchable.CommonRecordTestImpls(__name__).report_files() +launchable.CommonFlakeDetectionImpls(__name__).detect_flakes() diff --git a/launchable/utils/commands.py b/launchable/utils/commands.py index bd65a52ba..dde29c98d 100644 --- a/launchable/utils/commands.py +++ b/launchable/utils/commands.py @@ -8,7 +8,7 @@ class Command(Enum): RECORD_SESSION = 'RECORD_SESSION' SUBSET = 'SUBSET' COMMIT = 'COMMIT' - FLAKE_DETECTION = 'FLAKE_DETECTION' + DETECT_FLAKE = 'DETECT_FLAKE' def display_name(self): return self.value.lower().replace('_', ' ') diff --git a/tests/commands/test_flake_detection.py b/tests/commands/test_detect_flakes.py similarity index 63% rename from tests/commands/test_flake_detection.py rename to tests/commands/test_detect_flakes.py index 4589b8c1f..5266c837b 100644 --- a/tests/commands/test_flake_detection.py +++ b/tests/commands/test_detect_flakes.py @@ -7,7 +7,7 @@ from tests.cli_test_case import CliTestCase -class FlakeDetectionTest(CliTestCase): +class DetectFlakeTest(CliTestCase): @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_flake_detection_success(self): @@ -19,16 +19,15 @@ def test_flake_detection_success(self): } responses.add( responses.GET, - f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", json=mock_json_response, status=200, ) result = self.cli( - "retry", - "flake-detection", + "detect-flakes", "--session", self.session, - "--confidence", + "--retry-threshold", "high", "file", mix_stderr=False, @@ -37,22 +36,47 @@ def test_flake_detection_success(self): self.assertIn("test_flaky_1.py", result.stdout) self.assertIn("test_flaky_2.py", result.stdout) + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_without_retry_threshold_success(self): + mock_json_response = { + "testPaths": [ + [{"type": "file", "name": "test_flaky_1.py"}], + [{"type": "file", "name": "test_flaky_2.py"}], + ] + } + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", + json=mock_json_response, + status=200, + ) + result = self.cli( + "detect-flakes", + "--session", + self.session, + "file", + mix_stderr=False, + ) + self.assert_success(result) + self.assertIn("test_flaky_1.py", result.stdout) + self.assertIn("test_flaky_2.py", result.stdout) + @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_flake_detection_no_flakes(self): mock_json_response = {"testPaths": []} responses.add( responses.GET, - f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", json=mock_json_response, status=200, ) result = self.cli( - "retry", - "flake-detection", + "detect-flakes", "--session", self.session, - "--confidence", + "--retry-threshold", "low", "file", mix_stderr=False, @@ -65,15 +89,14 @@ def test_flake_detection_no_flakes(self): def test_flake_detection_api_error(self): responses.add( responses.GET, - f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", status=500, ) result = self.cli( - "retry", - "flake-detection", + "detect-flakes", "--session", self.session, - "--confidence", + "--retry-threshold", "medium", "file", mix_stderr=False, From 55c99e959aef6ecf57ef9041c58c9c6bef04a2b2 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Tue, 16 Sep 2025 17:12:27 +0900 Subject: [PATCH 17/46] Add detailed output for retrying tests in flake detection --- launchable/commands/detect_flakes.py | 4 ++++ launchable/test_runners/launchable.py | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/launchable/commands/detect_flakes.py b/launchable/commands/detect_flakes.py index 3fcad9321..8928975b0 100644 --- a/launchable/commands/detect_flakes.py +++ b/launchable/commands/detect_flakes.py @@ -6,6 +6,7 @@ from launchable.app import Application from launchable.commands.helper import find_or_create_session from launchable.commands.test_path_writer import TestPathWriter +from launchable.testpath import unparse_test_path from launchable.utils.click import ignorable_error from launchable.utils.env_keys import REPORT_ERROR_KEY from launchable.utils.launchable_client import LaunchableClient @@ -75,6 +76,9 @@ def run(self): test_paths = res.json().get("testPaths", []) if test_paths: self.print(test_paths) + click.echo("Trying to retry the following tests:", err=True) + for detail in res.json().get("testDetails", []): + click.echo(f"{detail.get('reason')}: {unparse_test_path(detail.get('fullTestPath'))}", err=True) except Exception as e: tracking_client.send_error_event( event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, diff --git a/launchable/test_runners/launchable.py b/launchable/test_runners/launchable.py index 33d60b71e..eb8cb2331 100644 --- a/launchable/test_runners/launchable.py +++ b/launchable/test_runners/launchable.py @@ -9,7 +9,6 @@ from launchable.commands.record.tests import tests as record_tests_cmd from launchable.commands.split_subset import split_subset as split_subset_cmd from launchable.commands.subset import subset as subset_cmd -from launchable.testpath import unparse_test_path def cmdname(m): @@ -177,7 +176,7 @@ class CommonFlakeDetectionImpls: def __init__( self, module_name, - formatter=unparse_test_path, + formatter=None, seperator="\n", ): self.cmdname = cmdname(module_name) From cd08b2158b7bcdc1626ac70da30bd1b2b06a9a83 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Sep 2025 22:40:18 +0000 Subject: [PATCH 18/46] [tagpr] prepare for the next release From ada5c052ddf01268daa738cdf443e56d53a3bcd8 Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Tue, 24 Jun 2025 18:29:01 +0900 Subject: [PATCH 19/46] fix: an error message in a test --- tests/commands/record/test_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/record/test_session.py b/tests/commands/record/test_session.py index b3206b802..50fb36e1d 100644 --- a/tests/commands/record/test_session.py +++ b/tests/commands/record/test_session.py @@ -65,7 +65,7 @@ def test_run_session_with_flavor(self): result = self.cli("record", "session", "--build", self.build_name, "--flavor", "only-key") self.assert_exit_code(result, 2) - self.assertIn("Expected a key-value pair formatted as --option key=value", result.output) + self.assertIn("Expected a key-value pair formatted as --option key=value, but got 'only-key'", result.output) @responses.activate @mock.patch.dict(os.environ, { From cd4cf90e861d8faafb596717809d59970bb7504c Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Tue, 24 Jun 2025 19:17:38 +0900 Subject: [PATCH 20/46] fix: no color on tests --- tests/cli_test_case.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/cli_test_case.py b/tests/cli_test_case.py index e79611cc5..8b962bc3a 100644 --- a/tests/cli_test_case.py +++ b/tests/cli_test_case.py @@ -216,7 +216,17 @@ def cli(self, *args, **kwargs) -> click.testing.Result: mix_stderr = kwargs['mix_stderr'] del kwargs['mix_stderr'] - return CliRunner(mix_stderr=mix_stderr).invoke(main, args, catch_exceptions=False, **kwargs) + # Disable rich colors for testing by setting the environment variable + import os + old_no_color = os.environ.get('NO_COLOR') + os.environ['NO_COLOR'] = '1' + try: + return CliRunner(mix_stderr=mix_stderr).invoke(main, args, catch_exceptions=False, **kwargs) + finally: + if old_no_color is None: + os.environ.pop('NO_COLOR', None) + else: + os.environ['NO_COLOR'] = old_no_color def assert_success(self, result: click.testing.Result): self.assert_exit_code(result, 0) From 627f3655218d3f1a37f7aad14f719bca3ddfa0da Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Wed, 25 Jun 2025 11:16:03 +0900 Subject: [PATCH 21/46] fix: test --- tests/commands/record/test_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/record/test_session.py b/tests/commands/record/test_session.py index 50fb36e1d..5c5833f6e 100644 --- a/tests/commands/record/test_session.py +++ b/tests/commands/record/test_session.py @@ -65,7 +65,7 @@ def test_run_session_with_flavor(self): result = self.cli("record", "session", "--build", self.build_name, "--flavor", "only-key") self.assert_exit_code(result, 2) - self.assertIn("Expected a key-value pair formatted as --option key=value, but got 'only-key'", result.output) + self.assertIn("but got 'only-key'", result.output) @responses.activate @mock.patch.dict(os.environ, { From fa8cabd3363d3b12b8740800922515f6d75fd64f Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Fri, 27 Jun 2025 10:26:30 +0900 Subject: [PATCH 22/46] test: revive test_click.py as test_typer.py --- tests/utils/test_typer.py | 59 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/utils/test_typer.py diff --git a/tests/utils/test_typer.py b/tests/utils/test_typer.py new file mode 100644 index 000000000..202c6f492 --- /dev/null +++ b/tests/utils/test_typer.py @@ -0,0 +1,59 @@ +import datetime +from datetime import timezone +from unittest import TestCase + +import typer +from dateutil.tz import tzlocal + +from launchable.utils.typer_types import (DATETIME_WITH_TZ, KEY_VALUE, convert_to_seconds, + validate_datetime_with_tz, validate_key_value) + + +class DurationTypeTest(TestCase): + def test_convert_to_seconds(self): + self.assertEqual(convert_to_seconds('30s'), 30) + self.assertEqual(convert_to_seconds('5m'), 300) + self.assertEqual(convert_to_seconds('1h30m'), 5400) + self.assertEqual(convert_to_seconds('1d10h15m'), 123300) + self.assertEqual(convert_to_seconds('15m 1d 10h'), 123300) + + with self.assertRaises(ValueError): + convert_to_seconds('1h30k') + + +class KeyValueTypeTest(TestCase): + def test_conversion(self): + # Test the validate_key_value function directly + self.assertEqual(validate_key_value('bar=zot'), ('bar', 'zot')) + self.assertEqual(validate_key_value('a=b'), ('a', 'b')) + self.assertEqual(validate_key_value('key:value'), ('key', 'value')) + + with self.assertRaises(typer.BadParameter): + validate_key_value('invalid') + + # Test the parser class + parser = KEY_VALUE + self.assertEqual(parser('bar=zot'), ('bar', 'zot')) + self.assertEqual(parser('a=b'), ('a', 'b')) + + +class TimestampTypeTest(TestCase): + def test_conversion(self): + # Test the validate_datetime_with_tz function directly + result1 = validate_datetime_with_tz('2023-10-01 12:00:00') + expected1 = datetime.datetime(2023, 10, 1, 12, 0, 0, tzinfo=tzlocal()) + self.assertEqual(result1, expected1) + + result2 = validate_datetime_with_tz('2023-10-01 20:00:00+00:00') + expected2 = datetime.datetime(2023, 10, 1, 20, 0, 0, tzinfo=timezone.utc) + self.assertEqual(result2, expected2) + + result3 = validate_datetime_with_tz('2023-10-01T20:00:00Z') + expected3 = datetime.datetime(2023, 10, 1, 20, 0, 0, tzinfo=timezone.utc) + self.assertEqual(result3, expected3) + + # Test the parser class + parser = DATETIME_WITH_TZ + result4 = parser('2023-10-01 12:00:00') + expected4 = datetime.datetime(2023, 10, 1, 12, 0, 0, tzinfo=tzlocal()) + self.assertEqual(result4, expected4) From 28e98f175b49ebea64296c4001b46d5f3b9c4bb8 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 19 Jun 2025 09:58:40 +0900 Subject: [PATCH 23/46] Groovy lang works on JVM but we haven't supported it as jvm_test_pattern so, fixed the regexp to support groovy From 051dde769efdfec2c1ce215cd3d70483df6650e8 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 19 Jun 2025 14:24:13 +0900 Subject: [PATCH 24/46] support groovy file in the maven profile From 6c60db2556a7a27ace1c52f8b65608b80d22e5ba Mon Sep 17 00:00:00 2001 From: Konboi Date: Mon, 23 Jun 2025 18:05:44 +0900 Subject: [PATCH 25/46] install latest version --- setup.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index d86cc5b35..176e2e11d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,9 @@ [metadata] -name = launchable +name = smart-tests author = Launchable, Inc. author_email = info@launchableinc.com license = Apache Software License v2 -description = Launchable CLI +description = Smart Tests CLI url = https://launchableinc.com/ long_description = file: README.md long_description_content_type = text/markdown @@ -30,7 +30,7 @@ setup_requires = setuptools-scm [options.entry_points] -console_scripts = launchable = launchable.__main__:main +console_scripts = smart-tests = smart_tests.__main__:main [options.package_data] -launchable = jar/exe_deploy.jar +smart_tests = jar/exe_deploy.jar From 21f783a9775ddc491b816ebe809d89a4e5da987f Mon Sep 17 00:00:00 2001 From: Konboi Date: Mon, 23 Jun 2025 18:09:03 +0900 Subject: [PATCH 26/46] from junitparser v4.0.0 returns JUnitXml instead of testsuite From 23849dd70f3cda6331dbcf389277bcaa15fa3e0b Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Fri, 13 Jun 2025 10:59:54 +0900 Subject: [PATCH 27/46] [LCHIB-612] Add a workaround for handling timezone abbreviations in dateutil From b95ee7612ba101453b37a117a035f88df314695b Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Wed, 25 Jun 2025 11:03:06 +0900 Subject: [PATCH 28/46] Use assertEqual instead of assertTrue and assertIn From 1e9521eef0dbf904f54dcba20dccae3114b729cc Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Wed, 25 Jun 2025 11:07:33 +0900 Subject: [PATCH 29/46] Add comment From 0d0311af8c157e57f809fed47f5242f875cf9f1f Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Fri, 27 Jun 2025 10:26:30 +0900 Subject: [PATCH 30/46] test: revive test_click.py as test_typer.py From 6e68f8efcd15c765bf897d8b442a9c505465e609 Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Fri, 4 Jul 2025 17:22:11 +0900 Subject: [PATCH 31/46] feature: replace LAUNCHABLE_ env in Java code --- .../ingest/commits/CommitIngester.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/launchableinc/ingest/commits/CommitIngester.java b/src/main/java/com/launchableinc/ingest/commits/CommitIngester.java index 2efff19c0..302b30759 100644 --- a/src/main/java/com/launchableinc/ingest/commits/CommitIngester.java +++ b/src/main/java/com/launchableinc/ingest/commits/CommitIngester.java @@ -88,18 +88,18 @@ public void setNoCommitMessage(boolean b) { private void parseConfiguration() throws CmdLineException { String apiToken = launchableToken; if (launchableToken == null) { - apiToken = System.getenv("LAUNCHABLE_TOKEN"); + apiToken = System.getenv("SMART_TESTS_TOKEN"); } if (apiToken == null || apiToken.isEmpty()) { if (System.getenv("GITHUB_ACTIONS") != null) { - String o = System.getenv("LAUNCHABLE_ORGANIZATION"); + String o = System.getenv("SMART_TESTS_ORGANIZATION"); if (org == null && o == null) { - throw new CmdLineException("LAUNCHABLE_ORGANIZATION env variable is not set"); + throw new CmdLineException("SMART_TESTS_ORGANIZATION env variable is not set"); } - String w = System.getenv("LAUNCHABLE_WORKSPACE"); + String w = System.getenv("SMART_TESTS_WORKSPACE"); if (ws == null && w == null) { - throw new CmdLineException("LAUNCHABLE_WORKSPACE env variable is not set"); + throw new CmdLineException("SMART_TESTS_WORKSPACE env variable is not set"); } if (org == null) { @@ -118,7 +118,7 @@ private void parseConfiguration() throws CmdLineException { return; } - throw new CmdLineException("LAUNCHABLE_TOKEN env variable is not set"); + throw new CmdLineException("SMART_TESTS_TOKEN env variable is not set"); } this.parseLaunchableToken(apiToken); @@ -163,11 +163,11 @@ private void parseLaunchableToken(String token) throws CmdLineException { if (token.startsWith("v1:")) { String[] v = token.split(":"); if (v.length != 3) { - throw new IllegalStateException("Malformed LAUNCHABLE_TOKEN"); + throw new IllegalStateException("Malformed SMART_TESTS_TOKEN"); } v = v[1].split("/"); if (v.length != 2) { - throw new IllegalStateException("Malformed LAUNCHABLE_TOKEN"); + throw new IllegalStateException("Malformed SMART_TESTS_TOKEN"); } // for backward compatibility, allow command line options to take precedence From 6f5d168387b8539a209e0f7c9a5d31fac3be3e48 Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Fri, 4 Jul 2025 21:43:24 +0900 Subject: [PATCH 32/46] test: fix tests --- tests/commands/record/test_session.py | 87 ++++++++++++++++++++++++++- tests/commands/test_api_error.py | 33 ++++++++-- 2 files changed, 111 insertions(+), 9 deletions(-) diff --git a/tests/commands/record/test_session.py b/tests/commands/record/test_session.py index 5c5833f6e..61c53a482 100644 --- a/tests/commands/record/test_session.py +++ b/tests/commands/record/test_session.py @@ -24,7 +24,18 @@ class SessionTest(CliTestCase): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_without_flavor(self): - result = self.cli("record", "session", "--build", self.build_name) + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + + result = self.cli("record", "session", "--build", self.build_name, "--session", self.session_name) self.assert_success(result) payload = json.loads(responses.calls[1].request.body.decode()) @@ -44,7 +55,19 @@ def test_run_session_without_flavor(self): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_with_flavor(self): + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + result = self.cli("record", "session", "--build", self.build_name, + "--session", self.session_name, "--flavor", "key=value", "--flavor", "k:v", "--flavor", "k e y = v a l u e") self.assert_success(result) @@ -63,7 +86,18 @@ def test_run_session_with_flavor(self): "timestamp": None, }, payload) - result = self.cli("record", "session", "--build", self.build_name, "--flavor", "only-key") + # Mock session name check for second call + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + + result = self.cli("record", "session", "--build", self.build_name, "--session", self.session_name, "--flavor", "only-key") self.assert_exit_code(result, 2) self.assertIn("but got 'only-key'", result.output) @@ -73,7 +107,18 @@ def test_run_session_with_flavor(self): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_with_observation(self): - result = self.cli("record", "session", "--build", self.build_name, "--observation") + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + + result = self.cli("record", "session", "--build", self.build_name, "--session", self.session_name, "--observation") self.assert_success(result) payload = json.loads(responses.calls[1].request.body.decode()) @@ -133,7 +178,19 @@ def test_run_session_with_session_name(self): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_with_lineage(self): + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + result = self.cli("record", "session", "--build", self.build_name, + "--session", self.session_name, "--lineage", "example-lineage") self.assert_success(result) @@ -154,7 +211,19 @@ def test_run_session_with_lineage(self): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_with_test_suite(self): + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + result = self.cli("record", "session", "--build", self.build_name, + "--session", self.session_name, "--test-suite", "example-test-suite") self.assert_success(result) @@ -175,7 +244,19 @@ def test_run_session_with_test_suite(self): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_with_timestamp(self): + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + result = self.cli("record", "session", "--build", self.build_name, + "--session", self.session_name, "--timestamp", "2023-10-01T12:00:00Z") self.assert_success(result) diff --git a/tests/commands/test_api_error.py b/tests/commands/test_api_error.py index 8a9249a35..69bc3dfe3 100644 --- a/tests/commands/test_api_error.py +++ b/tests/commands/test_api_error.py @@ -175,6 +175,16 @@ def test_record_build(self): @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_record_session(self): build = "internal_server_error" + # Mock session name check + responses.add( + responses.GET, + "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions/{session}".format( + base=get_base_url(), + org=self.organization, + ws=self.workspace, + build=build, + session=self.session_name), + status=404) responses.add( responses.POST, "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions".format( @@ -189,12 +199,22 @@ def test_record_session(self): base=get_base_url()), body=ReadTimeout("error")) - result = self.cli("record", "session", "--build", build) + result = self.cli("record", "session", "--build", build, "--session", self.session_name) self.assert_success(result) - # Since HTTPError is occurred outside of LaunchableClient, the count is 1. - self.assert_tracking_count(tracking=tracking, count=1) + # Since HTTPError is occurred outside of LaunchableClient, the count is 2 (one for GET check, one for POST). + self.assert_tracking_count(tracking=tracking, count=2) build = "not_found" + # Mock session name check + responses.add( + responses.GET, + "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions/{session}".format( + base=get_base_url(), + org=self.organization, + ws=self.workspace, + build=build, + session=self.session_name), + status=404) responses.add( responses.POST, "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions".format( @@ -209,13 +229,14 @@ def test_record_session(self): base=get_base_url()), body=ReadTimeout("error")) - result = self.cli("record", "session", "--build", build) + result = self.cli("record", "session", "--build", build, "--session", self.session_name) self.assert_exit_code(result, 1) self.assert_tracking_count(tracking=tracking, count=1) - responses.replace( + # Mock session name check with ReadTimeout error + responses.add( responses.GET, - "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_session_names/{}".format( + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( get_base_url(), self.organization, self.workspace, From aa5d166ea6c177c400ddcb3d970ef5c54367a1e6 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Fri, 25 Jul 2025 14:08:57 +0900 Subject: [PATCH 33/46] Enhance PytestJSONReportParser to handle user properties as JSON From 637659d27fabae6a41dce6f738b45c124c0f66aa Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 4 Aug 2025 19:57:42 -0700 Subject: [PATCH 34/46] Follow up fix to f85f624d3816716a220bcd4123edad30ce88babc The caller side was not removed. Oof. From 1703ea917156a3a1475503a5d5ed979a35a0e562 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Tue, 5 Aug 2025 10:49:27 -0700 Subject: [PATCH 35/46] Merge pull request #1044 from launchableinc/renovate/actions-attest-build-provenance-2.x Update actions/attest-build-provenance action to v2.4.0 From 355c794d8a4037ae9b40f6b78f8dd274f2915710 Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Tue, 19 Aug 2025 11:31:46 +0900 Subject: [PATCH 36/46] refactor: remove intermidiate documents From 8b51f211652a549689d07a106632f67bb7e83b8c Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Wed, 20 Aug 2025 15:12:36 +0900 Subject: [PATCH 37/46] fix: import paths --- tests/utils/test_fail_fast_mode.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/utils/test_fail_fast_mode.py b/tests/utils/test_fail_fast_mode.py index c150e5f8f..8fc2c8437 100644 --- a/tests/utils/test_fail_fast_mode.py +++ b/tests/utils/test_fail_fast_mode.py @@ -1,8 +1,8 @@ import io from contextlib import contextmanager, redirect_stderr -from launchable.utils.commands import Command -from launchable.utils.fail_fast_mode import FailFastModeValidateParams, fail_fast_mode_validate +from smart_tests.utils.commands import Command +from smart_tests.utils.fail_fast_mode import FailFastModeValidateParams, fail_fast_mode_validate from tests.cli_test_case import CliTestCase @@ -30,7 +30,7 @@ def test_fail_fast_mode_validate(self): @contextmanager def tmp_set_fail_fast_mode(enabled: bool): - from launchable.utils.fail_fast_mode import _fail_fast_mode_cache, set_fail_fast_mode + from smart_tests.utils.fail_fast_mode import _fail_fast_mode_cache, set_fail_fast_mode original = _fail_fast_mode_cache try: set_fail_fast_mode(enabled) From 3b31a0f17bd6ce80c2c241d58b9e4784fe62cebc Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Wed, 20 Aug 2025 16:06:32 +0900 Subject: [PATCH 38/46] test: add test_typer_types.py --- tests/utils/test_typer_types.py | 251 ++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 tests/utils/test_typer_types.py diff --git a/tests/utils/test_typer_types.py b/tests/utils/test_typer_types.py new file mode 100644 index 000000000..9b605e33e --- /dev/null +++ b/tests/utils/test_typer_types.py @@ -0,0 +1,251 @@ +import datetime +import sys +from datetime import timezone +from unittest import TestCase + +import typer +from dateutil.tz import tzlocal + +from smart_tests.utils.typer_types import (DATETIME_WITH_TZ, EMOJI, KEY_VALUE, DateTimeWithTimezone, Duration, Fraction, + KeyValue, Percentage, convert_to_seconds, emoji, parse_datetime_with_timezone, + parse_duration, parse_fraction, parse_key_value, parse_percentage, + validate_datetime_with_tz, validate_key_value, validate_past_datetime) + + +class PercentageTest(TestCase): + def test_parse_valid_percentage(self): + pct = parse_percentage("50%") + self.assertIsInstance(pct, Percentage) + self.assertEqual(pct.value, 0.5) + self.assertEqual(float(pct), 0.5) + self.assertEqual(str(pct), "50.0%") + + def test_parse_edge_cases(self): + # Test 0% and 100% + self.assertEqual(parse_percentage("0%").value, 0.0) + self.assertEqual(parse_percentage("100%").value, 1.0) + + # Test decimal percentages + self.assertEqual(parse_percentage("25.5%").value, 0.255) + + def test_parse_invalid_percentage_missing_percent(self): + orig_platform = sys.platform + try: + # Test Windows behavior + sys.platform = "win32" + with self.assertRaises(typer.BadParameter) as cm: + parse_percentage("50") + msg = str(cm.exception) + self.assertIn("Expected percentage like 50% but got '50'", msg) + self.assertIn("please write '50%%' to pass in '50%'", msg) + + # Test non-Windows behavior + sys.platform = "linux" + with self.assertRaises(typer.BadParameter) as cm: + parse_percentage("50") + msg = str(cm.exception) + self.assertIn("Expected percentage like 50% but got '50'", msg) + self.assertNotIn("please write '50%%' to pass in '50%'", msg) + finally: + sys.platform = orig_platform + + def test_parse_invalid_percentage_non_numeric(self): + with self.assertRaises(typer.BadParameter) as cm: + parse_percentage("abc%") + msg = str(cm.exception) + self.assertIn("Expected percentage like 50% but got 'abc%'", msg) + + def test_percentage_class_methods(self): + pct = Percentage(0.75) + self.assertEqual(str(pct), "75.0%") + self.assertEqual(float(pct), 0.75) + + +class DurationTest(TestCase): + def test_convert_to_seconds(self): + self.assertEqual(convert_to_seconds('30s'), 30) + self.assertEqual(convert_to_seconds('5m'), 300) + self.assertEqual(convert_to_seconds('1h30m'), 5400) + self.assertEqual(convert_to_seconds('1d10h15m'), 123300) + self.assertEqual(convert_to_seconds('15m 1d 10h'), 123300) + self.assertEqual(convert_to_seconds('1w'), 604800) # 7 days + + # Test numeric only + self.assertEqual(convert_to_seconds('3600'), 3600) + + def test_convert_to_seconds_invalid(self): + with self.assertRaises(ValueError): + convert_to_seconds('1h30k') + + def test_parse_duration(self): + duration = parse_duration("30s") + self.assertIsInstance(duration, Duration) + self.assertEqual(duration.seconds, 30) + self.assertEqual(float(duration), 30) + self.assertEqual(str(duration), "30.0s") + + def test_parse_duration_invalid(self): + # Note: convert_to_seconds returns 0.0 for invalid input instead of raising ValueError + # So parse_duration returns Duration(0.0) for invalid input + duration = parse_duration("invalid") + self.assertEqual(duration.seconds, 0.0) + + +class KeyValueTest(TestCase): + def test_parse_key_value_equals(self): + kv = parse_key_value("key=value") + self.assertIsInstance(kv, KeyValue) + self.assertEqual(kv.key, "key") + self.assertEqual(kv.value, "value") + self.assertEqual(str(kv), "key=value") + + # Test tuple-like behavior + self.assertEqual(kv[0], "key") + self.assertEqual(kv[1], "value") + self.assertEqual(list(kv), ["key", "value"]) + + def test_parse_key_value_colon(self): + kv = parse_key_value("key:value") + self.assertEqual(kv.key, "key") + self.assertEqual(kv.value, "value") + + def test_parse_key_value_with_spaces(self): + kv = parse_key_value(" key = value ") + self.assertEqual(kv.key, "key") + self.assertEqual(kv.value, "value") + + def test_parse_key_value_with_multiple_delimiters(self): + # Should split on first occurrence only + kv = parse_key_value("key=value=extra") + self.assertEqual(kv.key, "key") + self.assertEqual(kv.value, "value=extra") + + def test_parse_key_value_invalid(self): + with self.assertRaises(typer.BadParameter) as cm: + parse_key_value("invalid") + msg = str(cm.exception) + self.assertIn("Expected a key-value pair formatted as --option key=value, but got 'invalid'", msg) + + def test_validate_key_value_compat(self): + # Test backward compatibility function + result = validate_key_value("key=value") + self.assertEqual(result, ("key", "value")) + + def test_key_value_compat_function(self): + # Test the KEY_VALUE constant + result = KEY_VALUE("key=value") + self.assertEqual(result, ("key", "value")) + + +class FractionTest(TestCase): + def test_parse_fraction(self): + frac = parse_fraction("3/4") + self.assertIsInstance(frac, Fraction) + self.assertEqual(frac.numerator, 3) + self.assertEqual(frac.denominator, 4) + self.assertEqual(str(frac), "3/4") + self.assertEqual(float(frac), 0.75) + + # Test tuple-like behavior + self.assertEqual(frac[0], 3) + self.assertEqual(frac[1], 4) + self.assertEqual(list(frac), [3, 4]) + + def test_parse_fraction_with_spaces(self): + frac = parse_fraction(" 1 / 2 ") + self.assertEqual(frac.numerator, 1) + self.assertEqual(frac.denominator, 2) + + def test_parse_fraction_invalid(self): + with self.assertRaises(typer.BadParameter) as cm: + parse_fraction("invalid") + msg = str(cm.exception) + self.assertIn("Expected fraction like 1/2 but got 'invalid'", msg) + + def test_parse_fraction_invalid_numbers(self): + with self.assertRaises(typer.BadParameter): + parse_fraction("a/b") + + +class DateTimeWithTimezoneTest(TestCase): + def test_parse_datetime_with_timezone(self): + dt_str = "2023-10-01 12:00:00+00:00" + dt_obj = parse_datetime_with_timezone(dt_str) + self.assertIsInstance(dt_obj, DateTimeWithTimezone) + self.assertEqual(dt_obj.dt.year, 2023) + self.assertEqual(dt_obj.dt.month, 10) + self.assertEqual(dt_obj.dt.day, 1) + self.assertEqual(dt_obj.dt.hour, 12) + # dateutil.parser creates tzutc() which is equivalent to but not equal to timezone.utc + self.assertEqual(dt_obj.dt.utcoffset(), timezone.utc.utcoffset(None)) + + def test_parse_datetime_without_timezone(self): + dt_str = "2023-10-01 12:00:00" + dt_obj = parse_datetime_with_timezone(dt_str) + self.assertEqual(dt_obj.dt.tzinfo, tzlocal()) + + def test_parse_datetime_iso_format(self): + dt_str = "2023-10-01T20:00:00Z" + dt_obj = parse_datetime_with_timezone(dt_str) + # dateutil.parser creates tzutc() which is equivalent to but not equal to timezone.utc + self.assertEqual(dt_obj.dt.utcoffset(), timezone.utc.utcoffset(None)) + + def test_parse_datetime_invalid(self): + with self.assertRaises(typer.BadParameter) as cm: + parse_datetime_with_timezone("invalid") + msg = str(cm.exception) + self.assertIn("Expected datetime like 2023-10-01T12:00:00 but got 'invalid'", msg) + + def test_datetime_with_timezone_methods(self): + dt_obj = parse_datetime_with_timezone("2023-10-01T12:00:00Z") + self.assertEqual(dt_obj.datetime(), dt_obj.dt) + # Test string representation + self.assertIn("2023-10-01T12:00:00", str(dt_obj)) + + def test_validate_datetime_with_tz_compat(self): + # Test backward compatibility function + result = validate_datetime_with_tz("2023-10-01T12:00:00Z") + self.assertIsInstance(result, datetime.datetime) + # dateutil.parser creates tzutc() which is equivalent to but not equal to timezone.utc + self.assertEqual(result.utcoffset(), timezone.utc.utcoffset(None)) + + def test_datetime_with_tz_compat_function(self): + # Test the DATETIME_WITH_TZ constant + result = DATETIME_WITH_TZ("2023-10-01T12:00:00Z") + self.assertIsInstance(result, datetime.datetime) + + def test_validate_past_datetime(self): + # Test with None + self.assertIsNone(validate_past_datetime(None)) + + # Test with past datetime + past_dt = datetime.datetime(2020, 1, 1, tzinfo=tzlocal()) + self.assertEqual(validate_past_datetime(past_dt), past_dt) + + # Test with future datetime + future_dt = datetime.datetime(2030, 1, 1, tzinfo=tzlocal()) + with self.assertRaises(typer.BadParameter) as cm: + validate_past_datetime(future_dt) + msg = str(cm.exception) + self.assertIn("The provided timestamp must be in the past", msg) + + # Test with non-datetime object + with self.assertRaises(typer.BadParameter) as cm: + validate_past_datetime("not a datetime") + msg = str(cm.exception) + self.assertIn("Expected a datetime object", msg) + + +class EmojiTest(TestCase): + def test_emoji_function(self): + # Test with fallback + result = emoji("🎉", "!") + self.assertIn(result, ["🎉", "!"]) # Depends on system capability + + # Test without fallback + result = emoji("🎉") + self.assertIn(result, ["🎉", ""]) # Depends on system capability + + def test_emoji_constant(self): + # EMOJI should be a boolean + self.assertIsInstance(EMOJI, bool) From ca94e12066db234da025e9f46a5c331565d6aaf1 Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Wed, 20 Aug 2025 16:48:25 +0900 Subject: [PATCH 39/46] fix: test --- tests/commands/record/test_build.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/commands/record/test_build.py b/tests/commands/record/test_build.py index 6f6591146..95f5eb7f3 100644 --- a/tests/commands/record/test_build.py +++ b/tests/commands/record/test_build.py @@ -195,20 +195,20 @@ def test_commit_option_and_build_option(self): result = self.cli( "record", "build", + "--build", + self.build_name, "--no-commit-collection", "--commit", "A=abc12", "--branch", - "B=feature-yyy", - "--name", - self.build_name) + "B=feature-yyy") self.assert_success(result) payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal( { "buildNumber": "123", - "lineage": None, + "lineage": "B=feature-yyy", "commitHashes": [ { "repositoryName": "A", @@ -226,6 +226,8 @@ def test_commit_option_and_build_option(self): result = self.cli( "record", "build", + "--build", + self.build_name, "--no-commit-collection", "--commit", "A=abc12", @@ -234,9 +236,7 @@ def test_commit_option_and_build_option(self): "--commit", "B=56cde", "--branch", - "A=feature-xxx", - "--name", - self.build_name) + "A=feature-xxx") self.assert_success(result) payload = json.loads(responses.calls[1].request.body.decode()) From 2f30de0a38a513246683781bbf5214c76ab75cae Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Wed, 20 Aug 2025 19:45:13 +0900 Subject: [PATCH 40/46] fix: convert value --- launchable/commands/subset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index f93b0c3d6..bb415f719 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -491,17 +491,17 @@ def get_payload( if target is not None: payload["goal"] = { "type": "subset-by-percentage", - "percentage": target, + "percentage": float(target), } elif duration is not None: payload["goal"] = { "type": "subset-by-absolute-time", - "duration": duration, + "duration": float(duration), } elif confidence is not None: payload["goal"] = { "type": "subset-by-confidence", - "percentage": confidence + "percentage": float(confidence) } elif goal_spec is not None: payload["goal"] = { From 3dccc35ef740aef27dff5ab797d6df6c3fbfc2c9 Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Thu, 21 Aug 2025 15:38:44 +0900 Subject: [PATCH 41/46] fix: a test file --- tests/commands/record/test_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/record/test_session.py b/tests/commands/record/test_session.py index 61c53a482..2a18df06b 100644 --- a/tests/commands/record/test_session.py +++ b/tests/commands/record/test_session.py @@ -161,7 +161,7 @@ def test_run_session_with_session_name(self): result = self.cli("record", "session", "--build", self.build_name, "--session-name", self.session_name) self.assert_success(result) - payload = json.loads(responses.calls[5].request.body.decode()) + payload = json.loads(responses.calls[3].request.body.decode()) self.assert_json_orderless_equal({ "flavors": {}, "isObservation": False, From 60edb4cf4ccb337dacea0051f2c4571b60cfff70 Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Thu, 21 Aug 2025 16:12:39 +0900 Subject: [PATCH 42/46] fix: a test file --- tests/commands/record/test_build.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/commands/record/test_build.py b/tests/commands/record/test_build.py index 95f5eb7f3..a7aa69df7 100644 --- a/tests/commands/record/test_build.py +++ b/tests/commands/record/test_build.py @@ -191,7 +191,7 @@ def test_commit_option_and_build_option(self): }, payload) responses.calls.reset() - # case --commit option and --branch option but another one + # case --commit option and --repo-branch-map option with invalid repo result = self.cli( "record", "build", @@ -201,6 +201,8 @@ def test_commit_option_and_build_option(self): "--commit", "A=abc12", "--branch", + "main", + "--repo-branch-map", "B=feature-yyy") self.assert_success(result) @@ -208,7 +210,7 @@ def test_commit_option_and_build_option(self): self.assert_json_orderless_equal( { "buildNumber": "123", - "lineage": "B=feature-yyy", + "lineage": "main", "commitHashes": [ { "repositoryName": "A", @@ -231,11 +233,11 @@ def test_commit_option_and_build_option(self): "--no-commit-collection", "--commit", "A=abc12", - "--branch", + "--repo-branch-map", "B=feature-yyy", "--commit", "B=56cde", - "--branch", + "--repo-branch-map", "A=feature-xxx") self.assert_success(result) From 326ce0e2e2d4f45e672f45d0850e01cc3cf4592d Mon Sep 17 00:00:00 2001 From: ninjinkun Date: Thu, 21 Aug 2025 18:37:58 +0900 Subject: [PATCH 43/46] fix: tracking event count for reducing traking event in SmartTests CLI --- tests/commands/test_api_error.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/commands/test_api_error.py b/tests/commands/test_api_error.py index 69bc3dfe3..69602c85c 100644 --- a/tests/commands/test_api_error.py +++ b/tests/commands/test_api_error.py @@ -132,7 +132,7 @@ def test_record_build(self): self.assert_success(result) self.assertEqual(result.exception, None) # Since HTTPError is occurred outside of LaunchableClient, the count is 1. - self.assert_tracking_count(tracking=tracking, count=3) + self.assert_tracking_count(tracking=tracking, count=2) success_server.shutdown() thread.join(timeout=3) @@ -166,7 +166,7 @@ def test_record_build(self): self.assert_success(result) self.assertEqual(result.exception, None) # Since HTTPError is occurred outside of LaunchableClient, the count is 1. - self.assert_tracking_count(tracking=tracking, count=3) + self.assert_tracking_count(tracking=tracking, count=2) error_server.shutdown() thread.join(timeout=3) @@ -459,7 +459,7 @@ def test_all_workflow_when_server_down(self): self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=6) + self.assert_tracking_count(tracking=tracking, count=5) # set delete=False to solve the error `PermissionError: [Errno 13] Permission denied:` on Windows. with tempfile.NamedTemporaryFile(delete=False) as rest_file: @@ -476,12 +476,12 @@ def test_all_workflow_when_server_down(self): self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=9) + self.assert_tracking_count(tracking=tracking, count=7) result = self.cli("record", "tests", "--session", self.session, "minitest", str(self.test_files_dir) + "/") self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=13) + self.assert_tracking_count(tracking=tracking, count=9) def assert_tracking_count(self, tracking, count: int): # Prior to 3.6, `Response` object can't be obtained. From 8c32dace0b77580f052e0ae864735e7ddbf8fe31 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 4 Sep 2025 13:56:51 +0900 Subject: [PATCH 44/46] add util method to print error message and exit as 1 --- launchable/utils/exceptions.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/launchable/utils/exceptions.py b/launchable/utils/exceptions.py index 7504d0333..2fc2f4c12 100644 --- a/launchable/utils/exceptions.py +++ b/launchable/utils/exceptions.py @@ -1,4 +1,10 @@ # TODO: add cli-specific custom exceptions +import sys + +import typer + +from smart_tests.utils.tracking import Tracking, TrackingClient + class ParseSessionException(Exception): def __init__( @@ -20,3 +26,9 @@ def __init__( self.filename = filename self.message = "{message}: {filename}".format(message=message, filename=self.filename) super().__init__(self.message) + + +def print_error_and_die(msg: str, tracking_client: TrackingClient, event: Tracking.ErrorEvent): + typer.secho(msg, fg=typer.colors.RED, err=True) + tracking_client.send_error_event(event_name=event, stack_trace=msg) + sys.exit(1) From a4aaa7b7c7c884b852562484511f041f51a480dd Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 5 Sep 2025 10:37:14 -0700 Subject: [PATCH 45/46] [chore] report an error in subset call It looks like c41d9a6270d448168e722155b25ac04b7bdad5fa removed the `raise_for_status` check, without which error message from the server won't be reported, even as a warning. This breaks the subset call in case of the server failure, since `res.json()` looks as if it's returning an empty subset From f36cb29faf6550f43cba426108b5f5504128fa08 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Wed, 10 Sep 2025 14:57:18 +0900 Subject: [PATCH 46/46] Add tests --- tests/commands/test_flake_detection.py | 83 ++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/commands/test_flake_detection.py diff --git a/tests/commands/test_flake_detection.py b/tests/commands/test_flake_detection.py new file mode 100644 index 000000000..4589b8c1f --- /dev/null +++ b/tests/commands/test_flake_detection.py @@ -0,0 +1,83 @@ +import os +from unittest import mock + +import responses # type: ignore + +from launchable.utils.http_client import get_base_url +from tests.cli_test_case import CliTestCase + + +class FlakeDetectionTest(CliTestCase): + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_success(self): + mock_json_response = { + "testPaths": [ + [{"type": "file", "name": "test_flaky_1.py"}], + [{"type": "file", "name": "test_flaky_2.py"}], + ] + } + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + json=mock_json_response, + status=200, + ) + result = self.cli( + "retry", + "flake-detection", + "--session", + self.session, + "--confidence", + "high", + "file", + mix_stderr=False, + ) + self.assert_success(result) + self.assertIn("test_flaky_1.py", result.stdout) + self.assertIn("test_flaky_2.py", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_no_flakes(self): + mock_json_response = {"testPaths": []} + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + json=mock_json_response, + status=200, + ) + result = self.cli( + "retry", + "flake-detection", + "--session", + self.session, + "--confidence", + "low", + "file", + mix_stderr=False, + ) + self.assert_success(result) + self.assertEqual(result.stdout, "") + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_api_error(self): + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + status=500, + ) + result = self.cli( + "retry", + "flake-detection", + "--session", + self.session, + "--confidence", + "medium", + "file", + mix_stderr=False, + ) + self.assert_exit_code(result, 0) + self.assertIn("Error", result.stderr) + self.assertEqual(result.stdout, "")