diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index aa43036fa..a2852b692 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -2,7 +2,7 @@ name: e2e on: push: - branches: [main] + branches: [v1] workflow_dispatch: diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index aee24985c..f9357aead 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -6,12 +6,12 @@ name: Python package on: workflow_dispatch: push: - branches: [ main ] + branches: [ v1 ] paths-ignore: - 'WORKSPACE' - 'src/**' pull_request: - branches: [ main ] + branches: [ v1 ] paths-ignore: - 'WORKSPACE' - 'src/**' diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 17b2b2484..9ba369b50 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: push: branches: - - main + - v1 env: IMAGE_NAME: cloudbees/launchable diff --git a/.tagpr b/.tagpr index dc1d98b70..a508ee580 100644 --- a/.tagpr +++ b/.tagpr @@ -38,6 +38,6 @@ # [tagpr] vPrefix = true - releaseBranch = main + releaseBranch = v1 versionFile = - changelog = false diff --git a/launchable/__main__.py b/launchable/__main__.py index ef81dfd9a..34d8b5fa8 100644 --- a/launchable/__main__.py +++ b/launchable/__main__.py @@ -10,6 +10,7 @@ from launchable.app import Application from .commands.compare import compare +from .commands.detect_flakes import detect_flakes from .commands.inspect import inspect from .commands.record import record from .commands.split_subset import split_subset @@ -91,6 +92,7 @@ def main(ctx, log_level, plugin_dir, dry_run, skip_cert_verification): main.add_command(inspect) main.add_command(stats) main.add_command(compare) +main.add_command(detect_flakes, "detect-flakes") if __name__ == '__main__': main() diff --git a/launchable/commands/compare/subsets.py b/launchable/commands/compare/subsets.py index d55047256..c6162f4e0 100644 --- a/launchable/commands/compare/subsets.py +++ b/launchable/commands/compare/subsets.py @@ -47,4 +47,4 @@ def subsets(file_before, file_after): (before, after, f"{diff:+}" if isinstance(diff, int) else diff, test) for before, after, diff, test in rows ] - click.echo(tabulate(tabular_data, headers=headers, tablefmt="github")) + click.echo_via_pager(tabulate(tabular_data, headers=headers, tablefmt="github")) diff --git a/launchable/commands/detect_flakes.py b/launchable/commands/detect_flakes.py new file mode 100644 index 000000000..8928975b0 --- /dev/null +++ b/launchable/commands/detect_flakes.py @@ -0,0 +1,92 @@ +import os +import sys + +import click + +from launchable.app import Application +from launchable.commands.helper import find_or_create_session +from launchable.commands.test_path_writer import TestPathWriter +from launchable.testpath import unparse_test_path +from launchable.utils.click import ignorable_error +from launchable.utils.env_keys import REPORT_ERROR_KEY +from launchable.utils.launchable_client import LaunchableClient +from launchable.utils.tracking import Tracking, TrackingClient + +from ..utils.commands import Command + + +@click.group(help="Early flake detection") +@click.option( + '--session', + 'session', + help='In the format builds//test_sessions/', + type=str, + required=True +) +@click.option( + '--retry-threshold', + 'retry_threshold', + help='Throughness of how "flake" is detected', + type=click.Choice(['low', 'medium', 'high'], case_sensitive=False), + default='medium', + required=True, +) +@click.pass_context +def detect_flakes(ctx, retry_threshold, session): + tracking_client = TrackingClient(Command.DETECT_FLAKE, app=ctx.obj) + client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client, test_runner=ctx.invoked_subcommand) + session_id = None + try: + session_id = find_or_create_session( + context=ctx, + session=session, + build_name=None, + tracking_client=tracking_client + ) + except click.UsageError as e: + click.echo(click.style(str(e), fg="red"), err=True) + sys.exit(1) + except Exception as e: + tracking_client.send_error_event( + event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, + stack_trace=str(e), + ) + if os.getenv(REPORT_ERROR_KEY): + raise e + else: + click.echo(ignorable_error(e), err=True) + if session_id is None: + return + + class FlakeDetection(TestPathWriter): + def __init__(self, app: Application): + super(FlakeDetection, self).__init__(app=app) + + def run(self): + test_paths = [] + try: + res = client.request( + "get", + "detect-flake", + params={ + "confidence": retry_threshold.upper(), + "session-id": os.path.basename(session_id), + "test-runner": ctx.invoked_subcommand}) + res.raise_for_status() + test_paths = res.json().get("testPaths", []) + if test_paths: + self.print(test_paths) + click.echo("Trying to retry the following tests:", err=True) + for detail in res.json().get("testDetails", []): + click.echo(f"{detail.get('reason')}: {unparse_test_path(detail.get('fullTestPath'))}", err=True) + except Exception as e: + tracking_client.send_error_event( + event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, + stack_trace=str(e), + ) + if os.getenv(REPORT_ERROR_KEY): + raise e + else: + click.echo(ignorable_error(e), err=True) + + ctx.obj = FlakeDetection(app=ctx.obj) diff --git a/launchable/commands/helper.py b/launchable/commands/helper.py index ebd83e887..6606d518c 100644 --- a/launchable/commands/helper.py +++ b/launchable/commands/helper.py @@ -124,6 +124,8 @@ def find_or_create_session( session_id = read_session(saved_build_name) if session_id: _check_observation_mode_status(session_id, is_observation, tracking_client=tracking_client, app=context.obj) + if links: + click.echo(click.style("WARNING: --link option is ignored since session already exists."), err=True) return session_id context.invoke( diff --git a/launchable/commands/inspect/subset.py b/launchable/commands/inspect/subset.py index a13e98505..5ff0baeb1 100644 --- a/launchable/commands/inspect/subset.py +++ b/launchable/commands/inspect/subset.py @@ -65,7 +65,7 @@ def display(self): result._estimated_duration_sec, ] ) - click.echo(tabulate(rows, header, tablefmt="github", floatfmt=".2f")) + click.echo_via_pager(tabulate(rows, header, tablefmt="github", floatfmt=".2f")) class SubsetResultJSONDisplay(SubsetResultAbstractDisplay): diff --git a/launchable/commands/record/build.py b/launchable/commands/record/build.py index 321aceeff..e5dd3aa13 100644 --- a/launchable/commands/record/build.py +++ b/launchable/commands/record/build.py @@ -7,7 +7,7 @@ import click from tabulate import tabulate -from launchable.utils.link import CIRCLECI_KEY, GITHUB_ACTIONS_KEY, JENKINS_URL_KEY, LinkKind, capture_link +from launchable.utils.link import CIRCLECI_KEY, GITHUB_ACTIONS_KEY, JENKINS_URL_KEY, capture_links from launchable.utils.tracking import Tracking, TrackingClient from ...utils import subprocess @@ -316,14 +316,7 @@ def synthesize_workspaces() -> List[Workspace]: def send(ws: List[Workspace]) -> Optional[str]: # figure out all the CI links to capture def compute_links(): - _links = capture_link(os.environ) - for k, v in links: - _links.append({ - "title": k, - "url": v, - "kind": LinkKind.CUSTOM_LINK.name, - }) - return _links + return capture_links(links, os.environ) try: payload = { diff --git a/launchable/commands/record/session.py b/launchable/commands/record/session.py index b5376e1a0..d751baa9d 100644 --- a/launchable/commands/record/session.py +++ b/launchable/commands/record/session.py @@ -8,7 +8,7 @@ import click from launchable.utils.click import DATETIME_WITH_TZ, validate_past_datetime -from launchable.utils.link import LinkKind, capture_link +from launchable.utils.link import capture_links from launchable.utils.tracking import Tracking, TrackingClient from ...utils.click import KEY_VALUE @@ -181,25 +181,17 @@ def session( flavor_dict = dict(flavor) - payload = { - "flavors": flavor_dict, - "isObservation": is_observation, - "noBuild": is_no_build, - "lineage": lineage, - "testSuite": test_suite, - "timestamp": timestamp.isoformat() if timestamp else None, - } - - _links = capture_link(os.environ) - for link in links: - _links.append({ - "title": link[0], - "url": link[1], - "kind": LinkKind.CUSTOM_LINK.name, - }) - payload["links"] = _links - try: + payload = { + "flavors": flavor_dict, + "isObservation": is_observation, + "noBuild": is_no_build, + "lineage": lineage, + "testSuite": test_suite, + "timestamp": timestamp.isoformat() if timestamp else None, + "links": capture_links(links, os.environ), + } + sub_path = "builds/{}/test_sessions".format(build_name) res = client.request("post", sub_path, payload=payload) diff --git a/launchable/commands/record/tests.py b/launchable/commands/record/tests.py index 64415692a..f86ba9d03 100644 --- a/launchable/commands/record/tests.py +++ b/launchable/commands/record/tests.py @@ -229,6 +229,11 @@ def tests( is_no_build = False + if session and links: + warn_and_exit_if_fail_fast_mode( + "WARNING: `--link` and `--session` are set together.\n--link option can't be used with existing sessions." + ) + try: if is_no_build: session_id = "builds/{}/test_sessions/{}".format(NO_BUILD_BUILD_NAME, NO_BUILD_TEST_SESSION_ID) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index e0ec48e4a..bb415f719 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -207,6 +207,12 @@ help="get subset list from git managed files", is_flag=True, ) +@click.option( + "--use-case", + "use_case", + type=click.Choice(["one-commit", "feature-branch", "recurring"]), + hidden=True, # control PTS v2 test selection behavior. Non-committed, so hidden for now. +) @click.pass_context def subset( context: click.core.Context, @@ -235,6 +241,7 @@ def subset( prioritized_tests_mapping_file: Optional[TextIO] = None, test_suite: Optional[str] = None, is_get_tests_from_guess: bool = False, + use_case: Optional[str] = None, ): app = context.obj tracking_client = TrackingClient(Command.SUBSET, app=app) @@ -484,17 +491,17 @@ def get_payload( if target is not None: payload["goal"] = { "type": "subset-by-percentage", - "percentage": target, + "percentage": float(target), } elif duration is not None: payload["goal"] = { "type": "subset-by-absolute-time", - "duration": duration, + "duration": float(duration), } elif confidence is not None: payload["goal"] = { "type": "subset-by-confidence", - "percentage": confidence + "percentage": float(confidence) } elif goal_spec is not None: payload["goal"] = { @@ -513,6 +520,9 @@ def get_payload( if prioritized_tests_mapping_file: payload['prioritizedTestsMapping'] = json.load(prioritized_tests_mapping_file) + if use_case: + payload["changesUnderTest"] = use_case + return payload def _collect_potential_test_files(self): @@ -560,6 +570,7 @@ def request_subset(self) -> SubsetResult: # The status code 422 is returned when validation error of the test mapping file occurs. if res.status_code == 422: print_error_and_die("Error: {}".format(res.reason), Tracking.ErrorEvent.USER_ERROR) + res.raise_for_status() return SubsetResult.from_response(res.json()) except Exception as e: diff --git a/launchable/test_runners/bazel.py b/launchable/test_runners/bazel.py index 1904a926b..372716dd1 100644 --- a/launchable/test_runners/bazel.py +++ b/launchable/test_runners/bazel.py @@ -33,6 +33,8 @@ def subset(client): split_subset = launchable.CommonSplitSubsetImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).split_subset() +launchable.CommonFlakeDetectionImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).detect_flakes() + @click.argument('workspace', required=True) @click.option('--build-event-json', 'build_event_json_files', help="set file path generated by --build_event_json_file", diff --git a/launchable/test_runners/file.py b/launchable/test_runners/file.py index 8c36746fc..3b9a11549 100644 --- a/launchable/test_runners/file.py +++ b/launchable/test_runners/file.py @@ -52,3 +52,5 @@ def find_filename(): split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() + +launchable.CommonFlakeDetectionImpls(__name__).detect_flakes() diff --git a/launchable/test_runners/launchable.py b/launchable/test_runners/launchable.py index 92119326c..eb8cb2331 100644 --- a/launchable/test_runners/launchable.py +++ b/launchable/test_runners/launchable.py @@ -5,6 +5,7 @@ import click +from launchable.commands.detect_flakes import detect_flakes as detect_flakes_cmd from launchable.commands.record.tests import tests as record_tests_cmd from launchable.commands.split_subset import split_subset as split_subset_cmd from launchable.commands.subset import subset as subset_cmd @@ -43,6 +44,10 @@ def subset(f): record.tests = lambda f: wrap(f, record_tests_cmd) +def flake_detection(f): + return wrap(f, detect_flakes_cmd) + + def split_subset(f): return wrap(f, split_subset_cmd) @@ -165,3 +170,27 @@ def split_subset(client): client.run() return wrap(split_subset, split_subset_cmd, self.cmdname) + + +class CommonFlakeDetectionImpls: + def __init__( + self, + module_name, + formatter=None, + seperator="\n", + ): + self.cmdname = cmdname(module_name) + self._formatter = formatter + self._separator = seperator + + def detect_flakes(self): + def detect_flakes(client): + if self._formatter: + client.formatter = self._formatter + + if self._separator: + client.separator = self._separator + + client.run() + + return wrap(detect_flakes, detect_flakes_cmd, self.cmdname) diff --git a/launchable/test_runners/raw.py b/launchable/test_runners/raw.py index cf52d0245..565946b17 100644 --- a/launchable/test_runners/raw.py +++ b/launchable/test_runners/raw.py @@ -47,6 +47,8 @@ def subset(client, test_path_file): split_subset = launchable.CommonSplitSubsetImpls(__name__, formatter=unparse_test_path, seperator='\n').split_subset() +launchable.CommonFlakeDetectionImpls(__name__).detect_flakes() + @click.argument('test_result_files', required=True, type=click.Path(exists=True), nargs=-1) @launchable.record.tests diff --git a/launchable/test_runners/rspec.py b/launchable/test_runners/rspec.py index 15ffcc155..312137c80 100644 --- a/launchable/test_runners/rspec.py +++ b/launchable/test_runners/rspec.py @@ -3,3 +3,4 @@ subset = launchable.CommonSubsetImpls(__name__).scan_files('*_spec.rb') split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() record_tests = launchable.CommonRecordTestImpls(__name__).report_files() +launchable.CommonFlakeDetectionImpls(__name__).detect_flakes() diff --git a/launchable/utils/commands.py b/launchable/utils/commands.py index 6a8e357b6..dde29c98d 100644 --- a/launchable/utils/commands.py +++ b/launchable/utils/commands.py @@ -8,6 +8,7 @@ class Command(Enum): RECORD_SESSION = 'RECORD_SESSION' SUBSET = 'SUBSET' COMMIT = 'COMMIT' + DETECT_FLAKE = 'DETECT_FLAKE' def display_name(self): return self.value.lower().replace('_', ' ') diff --git a/launchable/utils/exceptions.py b/launchable/utils/exceptions.py index 7504d0333..2fc2f4c12 100644 --- a/launchable/utils/exceptions.py +++ b/launchable/utils/exceptions.py @@ -1,4 +1,10 @@ # TODO: add cli-specific custom exceptions +import sys + +import typer + +from smart_tests.utils.tracking import Tracking, TrackingClient + class ParseSessionException(Exception): def __init__( @@ -20,3 +26,9 @@ def __init__( self.filename = filename self.message = "{message}: {filename}".format(message=message, filename=self.filename) super().__init__(self.message) + + +def print_error_and_die(msg: str, tracking_client: TrackingClient, event: Tracking.ErrorEvent): + typer.secho(msg, fg=typer.colors.RED, err=True) + tracking_client.send_error_event(event_name=event, stack_trace=msg) + sys.exit(1) diff --git a/launchable/utils/link.py b/launchable/utils/link.py index 093221551..a017a1513 100644 --- a/launchable/utils/link.py +++ b/launchable/utils/link.py @@ -1,5 +1,8 @@ +import re from enum import Enum -from typing import Dict, List, Mapping +from typing import Dict, List, Mapping, Sequence, Tuple + +import click JENKINS_URL_KEY = 'JENKINS_URL' JENKINS_BUILD_URL_KEY = 'BUILD_URL' @@ -18,6 +21,8 @@ CIRCLECI_BUILD_NUM_KEY = 'CIRCLE_BUILD_NUM' CIRCLECI_JOB_KEY = 'CIRCLE_JOB' +GITHUB_PR_REGEX = re.compile(r"^https://github\.com/[^/]+/[^/]+/pull/\d+$") + class LinkKind(Enum): @@ -71,3 +76,81 @@ def capture_link(env: Mapping[str, str]) -> List[Dict[str, str]]: }) return links + + +def capture_links_from_options(link_options: Sequence[Tuple[str, str]]) -> List[Dict[str, str]]: + """ + Validate user-provided link options, inferring the kind when not explicitly specified. + + Each link option is expected in the format "kind|title=url" or "title=url". + If the kind is not provided, it infers the kind based on the URL pattern. + + Returns: + A list of dictionaries, where each dictionary contains the validated title, URL, and kind for each link. + + Raises: + click.UsageError: If an invalid kind is provided or URL doesn't match with the specified kind. + """ + links = [] + for k, url in link_options: + url = url.strip() + + # if k,v in format "kind|title=url" + if '|' in k: + kind, title = (part.strip() for part in k.split('|', 1)) + if kind not in _valid_kinds(): + msg = ("Invalid kind '{}' passed to --link option.\n" + "Supported kinds are: {}".format(kind, _valid_kinds())) + raise click.UsageError(click.style(msg, fg="red")) + + if not _url_matches_kind(url, kind): + msg = ("Invalid url '{}' passed to --link option.\n" + "URL doesn't match with the specified kind: {}".format(url, kind)) + raise click.UsageError(click.style(msg, fg="red")) + + # if k,v in format "title=url" + else: + kind = _infer_kind(url) + title = k.strip() + + links.append({ + "title": title, + "url": url, + "kind": kind, + }) + + return links + + +def capture_links(link_options: Sequence[Tuple[str, str]], env: Mapping[str, str]) -> List[Dict[str, str]]: + + links = capture_links_from_options(link_options) + + env_links = capture_link(env) + for env_link in env_links: + if not _has_kind(links, env_link['kind']): + links.append(env_link) + + return links + + +def _infer_kind(url: str) -> str: + if GITHUB_PR_REGEX.match(url): + return LinkKind.GITHUB_PULL_REQUEST.name + + return LinkKind.CUSTOM_LINK.name + + +def _has_kind(input_links: List[Dict[str, str]], kind: str) -> bool: + return any(link for link in input_links if link['kind'] == kind) + + +def _valid_kinds() -> List[str]: + return [kind.name for kind in LinkKind if kind != LinkKind.LINK_KIND_UNSPECIFIED] + + +def _url_matches_kind(url: str, kind: str) -> bool: + if kind == LinkKind.GITHUB_PULL_REQUEST.name: + return bool(GITHUB_PR_REGEX.match(url)) + + return True diff --git a/setup.cfg b/setup.cfg index d86cc5b35..176e2e11d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,9 @@ [metadata] -name = launchable +name = smart-tests author = Launchable, Inc. author_email = info@launchableinc.com license = Apache Software License v2 -description = Launchable CLI +description = Smart Tests CLI url = https://launchableinc.com/ long_description = file: README.md long_description_content_type = text/markdown @@ -30,7 +30,7 @@ setup_requires = setuptools-scm [options.entry_points] -console_scripts = launchable = launchable.__main__:main +console_scripts = smart-tests = smart_tests.__main__:main [options.package_data] -launchable = jar/exe_deploy.jar +smart_tests = jar/exe_deploy.jar diff --git a/src/main/java/com/launchableinc/ingest/commits/CommitIngester.java b/src/main/java/com/launchableinc/ingest/commits/CommitIngester.java index 2efff19c0..302b30759 100644 --- a/src/main/java/com/launchableinc/ingest/commits/CommitIngester.java +++ b/src/main/java/com/launchableinc/ingest/commits/CommitIngester.java @@ -88,18 +88,18 @@ public void setNoCommitMessage(boolean b) { private void parseConfiguration() throws CmdLineException { String apiToken = launchableToken; if (launchableToken == null) { - apiToken = System.getenv("LAUNCHABLE_TOKEN"); + apiToken = System.getenv("SMART_TESTS_TOKEN"); } if (apiToken == null || apiToken.isEmpty()) { if (System.getenv("GITHUB_ACTIONS") != null) { - String o = System.getenv("LAUNCHABLE_ORGANIZATION"); + String o = System.getenv("SMART_TESTS_ORGANIZATION"); if (org == null && o == null) { - throw new CmdLineException("LAUNCHABLE_ORGANIZATION env variable is not set"); + throw new CmdLineException("SMART_TESTS_ORGANIZATION env variable is not set"); } - String w = System.getenv("LAUNCHABLE_WORKSPACE"); + String w = System.getenv("SMART_TESTS_WORKSPACE"); if (ws == null && w == null) { - throw new CmdLineException("LAUNCHABLE_WORKSPACE env variable is not set"); + throw new CmdLineException("SMART_TESTS_WORKSPACE env variable is not set"); } if (org == null) { @@ -118,7 +118,7 @@ private void parseConfiguration() throws CmdLineException { return; } - throw new CmdLineException("LAUNCHABLE_TOKEN env variable is not set"); + throw new CmdLineException("SMART_TESTS_TOKEN env variable is not set"); } this.parseLaunchableToken(apiToken); @@ -163,11 +163,11 @@ private void parseLaunchableToken(String token) throws CmdLineException { if (token.startsWith("v1:")) { String[] v = token.split(":"); if (v.length != 3) { - throw new IllegalStateException("Malformed LAUNCHABLE_TOKEN"); + throw new IllegalStateException("Malformed SMART_TESTS_TOKEN"); } v = v[1].split("/"); if (v.length != 2) { - throw new IllegalStateException("Malformed LAUNCHABLE_TOKEN"); + throw new IllegalStateException("Malformed SMART_TESTS_TOKEN"); } // for backward compatibility, allow command line options to take precedence diff --git a/tests/cli_test_case.py b/tests/cli_test_case.py index e79611cc5..8b962bc3a 100644 --- a/tests/cli_test_case.py +++ b/tests/cli_test_case.py @@ -216,7 +216,17 @@ def cli(self, *args, **kwargs) -> click.testing.Result: mix_stderr = kwargs['mix_stderr'] del kwargs['mix_stderr'] - return CliRunner(mix_stderr=mix_stderr).invoke(main, args, catch_exceptions=False, **kwargs) + # Disable rich colors for testing by setting the environment variable + import os + old_no_color = os.environ.get('NO_COLOR') + os.environ['NO_COLOR'] = '1' + try: + return CliRunner(mix_stderr=mix_stderr).invoke(main, args, catch_exceptions=False, **kwargs) + finally: + if old_no_color is None: + os.environ.pop('NO_COLOR', None) + else: + os.environ['NO_COLOR'] = old_no_color def assert_success(self, result: click.testing.Result): self.assert_exit_code(result, 0) diff --git a/tests/commands/record/test_build.py b/tests/commands/record/test_build.py index 5d6940867..a7aa69df7 100644 --- a/tests/commands/record/test_build.py +++ b/tests/commands/record/test_build.py @@ -191,24 +191,26 @@ def test_commit_option_and_build_option(self): }, payload) responses.calls.reset() - # case --commit option and --branch option but another one + # case --commit option and --repo-branch-map option with invalid repo result = self.cli( "record", "build", + "--build", + self.build_name, "--no-commit-collection", "--commit", "A=abc12", "--branch", - "B=feature-yyy", - "--name", - self.build_name) + "main", + "--repo-branch-map", + "B=feature-yyy") self.assert_success(result) payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal( { "buildNumber": "123", - "lineage": None, + "lineage": "main", "commitHashes": [ { "repositoryName": "A", @@ -226,17 +228,17 @@ def test_commit_option_and_build_option(self): result = self.cli( "record", "build", + "--build", + self.build_name, "--no-commit-collection", "--commit", "A=abc12", - "--branch", + "--repo-branch-map", "B=feature-yyy", "--commit", "B=56cde", - "--branch", - "A=feature-xxx", - "--name", - self.build_name) + "--repo-branch-map", + "A=feature-xxx") self.assert_success(result) payload = json.loads(responses.calls[1].request.body.decode()) @@ -308,3 +310,50 @@ def test_with_timestamp(self, mock_check_output): }, payload) self.assertEqual(read_build(), self.build_name) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_build_with_links(self): + # Invalid kind + result = self.cli( + "record", + "build", + "--no-commit-collection", + "--link", + "UNKNOWN_KIND|PR=https://github.com/launchableinc/cli/pull/1", + "--name", + self.build_name) + self.assertIn("Invalid kind 'UNKNOWN_KIND' passed to --link option", result.output) + + # Invalid URL + result = self.cli( + "record", + "build", + "--no-commit-collection", + "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/1/files", + "--name", + self.build_name) + self.assertIn("Invalid url 'https://github.com/launchableinc/cli/pull/1/files' passed to --link option", result.output) + + # Infer kind + result = self.cli( + "record", + "build", + "--no-commit-collection", + "--link", + "PR=https://github.com/launchableinc/cli/pull/1", + "--name", + self.build_name) + self.assert_success(result) + + # Explicit kind + result = self.cli( + "record", + "build", + "--no-commit-collection", + "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/1", + "--name", + self.build_name) + self.assert_success(result) diff --git a/tests/commands/record/test_session.py b/tests/commands/record/test_session.py index ddac44e15..2a18df06b 100644 --- a/tests/commands/record/test_session.py +++ b/tests/commands/record/test_session.py @@ -5,6 +5,7 @@ import responses # type: ignore from launchable.utils.http_client import get_base_url +from launchable.utils.link import LinkKind from tests.cli_test_case import CliTestCase @@ -23,7 +24,18 @@ class SessionTest(CliTestCase): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_without_flavor(self): - result = self.cli("record", "session", "--build", self.build_name) + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + + result = self.cli("record", "session", "--build", self.build_name, "--session", self.session_name) self.assert_success(result) payload = json.loads(responses.calls[1].request.body.decode()) @@ -43,7 +55,19 @@ def test_run_session_without_flavor(self): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_with_flavor(self): + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + result = self.cli("record", "session", "--build", self.build_name, + "--session", self.session_name, "--flavor", "key=value", "--flavor", "k:v", "--flavor", "k e y = v a l u e") self.assert_success(result) @@ -62,9 +86,20 @@ def test_run_session_with_flavor(self): "timestamp": None, }, payload) - result = self.cli("record", "session", "--build", self.build_name, "--flavor", "only-key") + # Mock session name check for second call + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + + result = self.cli("record", "session", "--build", self.build_name, "--session", self.session_name, "--flavor", "only-key") self.assert_exit_code(result, 2) - self.assertIn("Expected a key-value pair formatted as --option key=value", result.output) + self.assertIn("but got 'only-key'", result.output) @responses.activate @mock.patch.dict(os.environ, { @@ -72,7 +107,18 @@ def test_run_session_with_flavor(self): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_with_observation(self): - result = self.cli("record", "session", "--build", self.build_name, "--observation") + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + + result = self.cli("record", "session", "--build", self.build_name, "--session", self.session_name, "--observation") self.assert_success(result) payload = json.loads(responses.calls[1].request.body.decode()) @@ -115,7 +161,7 @@ def test_run_session_with_session_name(self): result = self.cli("record", "session", "--build", self.build_name, "--session-name", self.session_name) self.assert_success(result) - payload = json.loads(responses.calls[5].request.body.decode()) + payload = json.loads(responses.calls[3].request.body.decode()) self.assert_json_orderless_equal({ "flavors": {}, "isObservation": False, @@ -132,7 +178,19 @@ def test_run_session_with_session_name(self): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_with_lineage(self): + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + result = self.cli("record", "session", "--build", self.build_name, + "--session", self.session_name, "--lineage", "example-lineage") self.assert_success(result) @@ -153,7 +211,19 @@ def test_run_session_with_lineage(self): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_with_test_suite(self): + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + result = self.cli("record", "session", "--build", self.build_name, + "--session", self.session_name, "--test-suite", "example-test-suite") self.assert_success(result) @@ -174,7 +244,19 @@ def test_run_session_with_test_suite(self): 'LANG': 'C.UTF-8', }, clear=True) def test_run_session_with_timestamp(self): + # Mock session name check + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name, + self.session_name), + status=404) + result = self.cli("record", "session", "--build", self.build_name, + "--session", self.session_name, "--timestamp", "2023-10-01T12:00:00Z") self.assert_success(result) @@ -188,3 +270,86 @@ def test_run_session_with_timestamp(self): "testSuite": None, "timestamp": "2023-10-01T12:00:00+00:00", }, payload) + + @responses.activate + @mock.patch.dict(os.environ, { + "LAUNCHABLE_TOKEN": CliTestCase.launchable_token, + 'LANG': 'C.UTF-8', + "GITHUB_PULL_REQUEST_URL": "https://github.com/launchableinc/cli/pull/1", + }, clear=True) + def test_run_session_with_links(self): + # Endpoint to assert + endpoint = "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name) + + # Capture from environment + result = self.cli("record", "session", "--build", self.build_name) + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 0).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "", + "url": "https://github.com/launchableinc/cli/pull/1", + }], payload["links"]) + + # Priority check + result = self.cli("record", "session", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 1).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Infer kind + result = self.cli("record", "session", "--build", self.build_name, "--link", + "PR=https://github.com/launchableinc/cli/pull/2") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 2).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Explicit kind + result = self.cli("record", "session", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 3).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Multiple kinds + result = self.cli("record", "session", "--build", self.build_name, "--link", + "GITHUB_ACTIONS|=https://github.com/launchableinc/mothership/actions/runs/3747451612") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 4).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_ACTIONS.name, + "title": "", + "url": "https://github.com/launchableinc/mothership/actions/runs/3747451612", + }, + { + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "", + "url": "https://github.com/launchableinc/cli/pull/1", + }], payload["links"]) + + # Invalid kind + result = self.cli("record", "session", "--build", self.build_name, "--link", + "UNKNOWN_KIND|PR=https://github.com/launchableinc/cli/pull/2") + self.assertIn("Invalid kind 'UNKNOWN_KIND' passed to --link option", result.output) + + # Invalid URL + result = self.cli("record", "session", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2/files") + self.assertIn("Invalid url 'https://github.com/launchableinc/cli/pull/2/files' passed to --link option", result.output) diff --git a/tests/commands/record/test_tests.py b/tests/commands/record/test_tests.py index 1e8c122e0..10125e119 100644 --- a/tests/commands/record/test_tests.py +++ b/tests/commands/record/test_tests.py @@ -9,6 +9,7 @@ from launchable.commands.record.tests import INVALID_TIMESTAMP, parse_launchable_timeformat from launchable.utils.http_client import get_base_url +from launchable.utils.link import LinkKind from launchable.utils.no_build import NO_BUILD_BUILD_NAME, NO_BUILD_TEST_SESSION_ID from launchable.utils.session import write_build, write_session from tests.cli_test_case import CliTestCase @@ -107,3 +108,106 @@ def test_when_total_test_duration_zero(self): self.assert_success(result) self.assertIn("Total test duration is 0.", result.output) + + @responses.activate + @mock.patch.dict(os.environ, { + "LAUNCHABLE_TOKEN": CliTestCase.launchable_token, + "GITHUB_PULL_REQUEST_URL": "https://github.com/launchableinc/cli/pull/1", + }, clear=True) + def test_with_links(self): + # Endpoint to assert + endpoint = "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions".format( + get_base_url(), + self.organization, + self.workspace, + self.build_name) + + # Capture from environment + write_build(self.build_name) + result = self.cli("record", "tests", "--build", self.build_name, 'maven', str(self.report_files_dir) + "**/reports/") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 0).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "", + "url": "https://github.com/launchableinc/cli/pull/1", + }], payload["links"]) + + # Priority check + write_build(self.build_name) + result = self.cli("record", "tests", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2", 'maven', + str(self.report_files_dir) + "**/reports/") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 1).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Infer kind + write_build(self.build_name) + result = self.cli("record", + "tests", + "--build", + self.build_name, + "--link", + "PR=https://github.com/launchableinc/cli/pull/2", + 'maven', + str(self.report_files_dir) + "**/reports/") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 2).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Explicit kind + write_build(self.build_name) + result = self.cli("record", "tests", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2", 'maven', + str(self.report_files_dir) + "**/reports/") + self.assert_success(result) + payload = json.loads(self.find_request(endpoint, 3).request.body.decode()) + self.assertEqual([{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }], payload["links"]) + + # Invalid kind + write_build(self.build_name) + result = self.cli("record", + "tests", + "--build", + self.build_name, + "--link", + "UNKNOWN_KIND|PR=https://github.com/launchableinc/cli/pull/2", + 'maven', + str(self.report_files_dir) + "**/reports/") + self.assertIn("Invalid kind 'UNKNOWN_KIND' passed to --link option", result.output) + + # Invalid URL + write_build(self.build_name) + result = self.cli("record", "tests", "--build", self.build_name, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2/files", 'maven', + str(self.report_files_dir) + "**/reports/") + self.assertIn("Invalid url 'https://github.com/launchableinc/cli/pull/2/files' passed to --link option", result.output) + + # With --session flag + write_session(self.build_name, self.session) + result = self.cli("record", "tests", "--session", self.session, "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2/files", 'maven', + str(self.report_files_dir) + "**/reports/") + self.assertIn("WARNING: `--link` and `--session` are set together", result.output) + self.assert_success(result) + + # Existing session + write_session(self.build_name, self.session) + result = self.cli("record", "tests", "--link", + "GITHUB_PULL_REQUEST|PR=https://github.com/launchableinc/cli/pull/2/files", 'maven', + str(self.report_files_dir) + "**/reports/") + self.assertIn("WARNING: --link option is ignored since session already exists", result.output) + self.assert_success(result) diff --git a/tests/commands/test_api_error.py b/tests/commands/test_api_error.py index 7c8cd16fd..69602c85c 100644 --- a/tests/commands/test_api_error.py +++ b/tests/commands/test_api_error.py @@ -3,6 +3,7 @@ import platform import tempfile import threading +import time from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path from unittest import mock @@ -131,7 +132,7 @@ def test_record_build(self): self.assert_success(result) self.assertEqual(result.exception, None) # Since HTTPError is occurred outside of LaunchableClient, the count is 1. - self.assert_tracking_count(tracking=tracking, count=3) + self.assert_tracking_count(tracking=tracking, count=2) success_server.shutdown() thread.join(timeout=3) @@ -165,7 +166,7 @@ def test_record_build(self): self.assert_success(result) self.assertEqual(result.exception, None) # Since HTTPError is occurred outside of LaunchableClient, the count is 1. - self.assert_tracking_count(tracking=tracking, count=3) + self.assert_tracking_count(tracking=tracking, count=2) error_server.shutdown() thread.join(timeout=3) @@ -174,6 +175,16 @@ def test_record_build(self): @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_record_session(self): build = "internal_server_error" + # Mock session name check + responses.add( + responses.GET, + "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions/{session}".format( + base=get_base_url(), + org=self.organization, + ws=self.workspace, + build=build, + session=self.session_name), + status=404) responses.add( responses.POST, "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions".format( @@ -188,12 +199,22 @@ def test_record_session(self): base=get_base_url()), body=ReadTimeout("error")) - result = self.cli("record", "session", "--build", build) + result = self.cli("record", "session", "--build", build, "--session", self.session_name) self.assert_success(result) - # Since HTTPError is occurred outside of LaunchableClient, the count is 1. - self.assert_tracking_count(tracking=tracking, count=1) + # Since HTTPError is occurred outside of LaunchableClient, the count is 2 (one for GET check, one for POST). + self.assert_tracking_count(tracking=tracking, count=2) build = "not_found" + # Mock session name check + responses.add( + responses.GET, + "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions/{session}".format( + base=get_base_url(), + org=self.organization, + ws=self.workspace, + build=build, + session=self.session_name), + status=404) responses.add( responses.POST, "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions".format( @@ -208,13 +229,14 @@ def test_record_session(self): base=get_base_url()), body=ReadTimeout("error")) - result = self.cli("record", "session", "--build", build) + result = self.cli("record", "session", "--build", build, "--session", self.session_name) self.assert_exit_code(result, 1) self.assert_tracking_count(tracking=tracking, count=1) - responses.replace( + # Mock session name check with ReadTimeout error + responses.add( responses.GET, - "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_session_names/{}".format( + "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}".format( get_base_url(), self.organization, self.workspace, @@ -437,7 +459,7 @@ def test_all_workflow_when_server_down(self): self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=6) + self.assert_tracking_count(tracking=tracking, count=5) # set delete=False to solve the error `PermissionError: [Errno 13] Permission denied:` on Windows. with tempfile.NamedTemporaryFile(delete=False) as rest_file: @@ -454,14 +476,21 @@ def test_all_workflow_when_server_down(self): self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=9) + self.assert_tracking_count(tracking=tracking, count=7) result = self.cli("record", "tests", "--session", self.session, "minitest", str(self.test_files_dir) + "/") self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=13) + self.assert_tracking_count(tracking=tracking, count=9) def assert_tracking_count(self, tracking, count: int): # Prior to 3.6, `Response` object can't be obtained. if compare_version([int(x) for x in platform.python_version().split('.')], [3, 7]) >= 0: - assert tracking.call_count == count + # Sometimes, `tracking.call_count` is not updated immediately. So, wait a moment until it is updated. + attempt = 0 + while tracking.call_count < count: + time.sleep(0.1) + attempt += 1 + if attempt > 10: + break + self.assertEqual(tracking.call_count, count) diff --git a/tests/commands/test_detect_flakes.py b/tests/commands/test_detect_flakes.py new file mode 100644 index 000000000..5266c837b --- /dev/null +++ b/tests/commands/test_detect_flakes.py @@ -0,0 +1,106 @@ +import os +from unittest import mock + +import responses # type: ignore + +from launchable.utils.http_client import get_base_url +from tests.cli_test_case import CliTestCase + + +class DetectFlakeTest(CliTestCase): + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_success(self): + mock_json_response = { + "testPaths": [ + [{"type": "file", "name": "test_flaky_1.py"}], + [{"type": "file", "name": "test_flaky_2.py"}], + ] + } + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", + json=mock_json_response, + status=200, + ) + result = self.cli( + "detect-flakes", + "--session", + self.session, + "--retry-threshold", + "high", + "file", + mix_stderr=False, + ) + self.assert_success(result) + self.assertIn("test_flaky_1.py", result.stdout) + self.assertIn("test_flaky_2.py", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_without_retry_threshold_success(self): + mock_json_response = { + "testPaths": [ + [{"type": "file", "name": "test_flaky_1.py"}], + [{"type": "file", "name": "test_flaky_2.py"}], + ] + } + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", + json=mock_json_response, + status=200, + ) + result = self.cli( + "detect-flakes", + "--session", + self.session, + "file", + mix_stderr=False, + ) + self.assert_success(result) + self.assertIn("test_flaky_1.py", result.stdout) + self.assertIn("test_flaky_2.py", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_no_flakes(self): + mock_json_response = {"testPaths": []} + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", + json=mock_json_response, + status=200, + ) + result = self.cli( + "detect-flakes", + "--session", + self.session, + "--retry-threshold", + "low", + "file", + mix_stderr=False, + ) + self.assert_success(result) + self.assertEqual(result.stdout, "") + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_api_error(self): + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", + status=500, + ) + result = self.cli( + "detect-flakes", + "--session", + self.session, + "--retry-threshold", + "medium", + "file", + mix_stderr=False, + ) + self.assert_exit_code(result, 0) + self.assertIn("Error", result.stderr) + self.assertEqual(result.stdout, "") diff --git a/tests/commands/test_flake_detection.py b/tests/commands/test_flake_detection.py new file mode 100644 index 000000000..4589b8c1f --- /dev/null +++ b/tests/commands/test_flake_detection.py @@ -0,0 +1,83 @@ +import os +from unittest import mock + +import responses # type: ignore + +from launchable.utils.http_client import get_base_url +from tests.cli_test_case import CliTestCase + + +class FlakeDetectionTest(CliTestCase): + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_success(self): + mock_json_response = { + "testPaths": [ + [{"type": "file", "name": "test_flaky_1.py"}], + [{"type": "file", "name": "test_flaky_2.py"}], + ] + } + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + json=mock_json_response, + status=200, + ) + result = self.cli( + "retry", + "flake-detection", + "--session", + self.session, + "--confidence", + "high", + "file", + mix_stderr=False, + ) + self.assert_success(result) + self.assertIn("test_flaky_1.py", result.stdout) + self.assertIn("test_flaky_2.py", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_no_flakes(self): + mock_json_response = {"testPaths": []} + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + json=mock_json_response, + status=200, + ) + result = self.cli( + "retry", + "flake-detection", + "--session", + self.session, + "--confidence", + "low", + "file", + mix_stderr=False, + ) + self.assert_success(result) + self.assertEqual(result.stdout, "") + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_api_error(self): + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + status=500, + ) + result = self.cli( + "retry", + "flake-detection", + "--session", + self.session, + "--confidence", + "medium", + "file", + mix_stderr=False, + ) + self.assert_exit_code(result, 0) + self.assertIn("Error", result.stderr) + self.assertEqual(result.stdout, "") diff --git a/tests/utils/test_fail_fast_mode.py b/tests/utils/test_fail_fast_mode.py index c150e5f8f..8fc2c8437 100644 --- a/tests/utils/test_fail_fast_mode.py +++ b/tests/utils/test_fail_fast_mode.py @@ -1,8 +1,8 @@ import io from contextlib import contextmanager, redirect_stderr -from launchable.utils.commands import Command -from launchable.utils.fail_fast_mode import FailFastModeValidateParams, fail_fast_mode_validate +from smart_tests.utils.commands import Command +from smart_tests.utils.fail_fast_mode import FailFastModeValidateParams, fail_fast_mode_validate from tests.cli_test_case import CliTestCase @@ -30,7 +30,7 @@ def test_fail_fast_mode_validate(self): @contextmanager def tmp_set_fail_fast_mode(enabled: bool): - from launchable.utils.fail_fast_mode import _fail_fast_mode_cache, set_fail_fast_mode + from smart_tests.utils.fail_fast_mode import _fail_fast_mode_cache, set_fail_fast_mode original = _fail_fast_mode_cache try: set_fail_fast_mode(enabled) diff --git a/tests/utils/test_http_client.py b/tests/utils/test_http_client.py index a565e2286..aadb886ce 100644 --- a/tests/utils/test_http_client.py +++ b/tests/utils/test_http_client.py @@ -2,8 +2,6 @@ import platform from unittest import TestCase, mock -from requests import Session - from launchable.utils.http_client import _HttpClient from launchable.version import __version__ @@ -45,19 +43,3 @@ def test_header(self): "dummy", ), }) - - def test_reason(self): - '''make sure we correctly propagate error message from the server''' - - # use new session to disable retry - cli = _HttpClient(session=Session()) - # /error is an actual endpoint that exists on our service to test the behavior - res = cli.request("GET", "intake/error") - self.assertEqual(res.status_code, 500) - self.assertEqual(res.reason, "Welp") - - try: - res.raise_for_status() - self.fail("should have raised") - except Exception as e: - self.assertIn("Welp", str(e)) diff --git a/tests/utils/test_link.py b/tests/utils/test_link.py index 3ee302b1c..b7527bb67 100644 --- a/tests/utils/test_link.py +++ b/tests/utils/test_link.py @@ -1,6 +1,8 @@ from unittest import TestCase -from launchable.utils.link import LinkKind, capture_link +import click + +from launchable.utils.link import LinkKind, capture_link, capture_links, capture_links_from_options class LinkTest(TestCase): @@ -44,3 +46,53 @@ def test_circleci(self): "title": "job (234)", "url": "https://circleci.com/build/123", }]) + + def test_capture_links_from_options(self): + # Invalid kind + link_options = [("INVALID_KIND|PR", "https://github.com/launchableinc/cli/pull/1")] + with self.assertRaises(click.UsageError): + capture_links_from_options(link_options) + + # Invalid URL + link_options = [("GITHUB_PULL_REQUEST|PR", "https://github.com/launchableinc/cli/pull/1/files")] + with self.assertRaises(click.UsageError): + capture_links_from_options(link_options) + + # Infer kind + link_options = [("PR", "https://github.com/launchableinc/cli/pull/1")] + self.assertEqual(capture_links_from_options(link_options), [{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/1", + }]) + + # Explicit kind + link_options = [("GITHUB_PULL_REQUEST|PR", "https://github.com/launchableinc/cli/pull/1")] + self.assertEqual(capture_links_from_options(link_options), [{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/1", + }]) + + def test_capture_links(self): + # Capture from environment + envs = { + "GITHUB_PULL_REQUEST_URL": "https://github.com/launchableinc/cli/pull/1" + } + link_options = [] + self.assertEqual(capture_links(link_options, envs), [{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "", + "url": "https://github.com/launchableinc/cli/pull/1", + }]) + + # Priority check + envs = { + "GITHUB_PULL_REQUEST_URL": "https://github.com/launchableinc/cli/pull/1" + } + link_options = [("GITHUB_PULL_REQUEST|PR", "https://github.com/launchableinc/cli/pull/2")] + self.assertEqual(capture_links(link_options, envs), [{ + "kind": LinkKind.GITHUB_PULL_REQUEST.name, + "title": "PR", + "url": "https://github.com/launchableinc/cli/pull/2", + }]) diff --git a/tests/utils/test_typer.py b/tests/utils/test_typer.py new file mode 100644 index 000000000..202c6f492 --- /dev/null +++ b/tests/utils/test_typer.py @@ -0,0 +1,59 @@ +import datetime +from datetime import timezone +from unittest import TestCase + +import typer +from dateutil.tz import tzlocal + +from launchable.utils.typer_types import (DATETIME_WITH_TZ, KEY_VALUE, convert_to_seconds, + validate_datetime_with_tz, validate_key_value) + + +class DurationTypeTest(TestCase): + def test_convert_to_seconds(self): + self.assertEqual(convert_to_seconds('30s'), 30) + self.assertEqual(convert_to_seconds('5m'), 300) + self.assertEqual(convert_to_seconds('1h30m'), 5400) + self.assertEqual(convert_to_seconds('1d10h15m'), 123300) + self.assertEqual(convert_to_seconds('15m 1d 10h'), 123300) + + with self.assertRaises(ValueError): + convert_to_seconds('1h30k') + + +class KeyValueTypeTest(TestCase): + def test_conversion(self): + # Test the validate_key_value function directly + self.assertEqual(validate_key_value('bar=zot'), ('bar', 'zot')) + self.assertEqual(validate_key_value('a=b'), ('a', 'b')) + self.assertEqual(validate_key_value('key:value'), ('key', 'value')) + + with self.assertRaises(typer.BadParameter): + validate_key_value('invalid') + + # Test the parser class + parser = KEY_VALUE + self.assertEqual(parser('bar=zot'), ('bar', 'zot')) + self.assertEqual(parser('a=b'), ('a', 'b')) + + +class TimestampTypeTest(TestCase): + def test_conversion(self): + # Test the validate_datetime_with_tz function directly + result1 = validate_datetime_with_tz('2023-10-01 12:00:00') + expected1 = datetime.datetime(2023, 10, 1, 12, 0, 0, tzinfo=tzlocal()) + self.assertEqual(result1, expected1) + + result2 = validate_datetime_with_tz('2023-10-01 20:00:00+00:00') + expected2 = datetime.datetime(2023, 10, 1, 20, 0, 0, tzinfo=timezone.utc) + self.assertEqual(result2, expected2) + + result3 = validate_datetime_with_tz('2023-10-01T20:00:00Z') + expected3 = datetime.datetime(2023, 10, 1, 20, 0, 0, tzinfo=timezone.utc) + self.assertEqual(result3, expected3) + + # Test the parser class + parser = DATETIME_WITH_TZ + result4 = parser('2023-10-01 12:00:00') + expected4 = datetime.datetime(2023, 10, 1, 12, 0, 0, tzinfo=tzlocal()) + self.assertEqual(result4, expected4) diff --git a/tests/utils/test_typer_types.py b/tests/utils/test_typer_types.py new file mode 100644 index 000000000..9b605e33e --- /dev/null +++ b/tests/utils/test_typer_types.py @@ -0,0 +1,251 @@ +import datetime +import sys +from datetime import timezone +from unittest import TestCase + +import typer +from dateutil.tz import tzlocal + +from smart_tests.utils.typer_types import (DATETIME_WITH_TZ, EMOJI, KEY_VALUE, DateTimeWithTimezone, Duration, Fraction, + KeyValue, Percentage, convert_to_seconds, emoji, parse_datetime_with_timezone, + parse_duration, parse_fraction, parse_key_value, parse_percentage, + validate_datetime_with_tz, validate_key_value, validate_past_datetime) + + +class PercentageTest(TestCase): + def test_parse_valid_percentage(self): + pct = parse_percentage("50%") + self.assertIsInstance(pct, Percentage) + self.assertEqual(pct.value, 0.5) + self.assertEqual(float(pct), 0.5) + self.assertEqual(str(pct), "50.0%") + + def test_parse_edge_cases(self): + # Test 0% and 100% + self.assertEqual(parse_percentage("0%").value, 0.0) + self.assertEqual(parse_percentage("100%").value, 1.0) + + # Test decimal percentages + self.assertEqual(parse_percentage("25.5%").value, 0.255) + + def test_parse_invalid_percentage_missing_percent(self): + orig_platform = sys.platform + try: + # Test Windows behavior + sys.platform = "win32" + with self.assertRaises(typer.BadParameter) as cm: + parse_percentage("50") + msg = str(cm.exception) + self.assertIn("Expected percentage like 50% but got '50'", msg) + self.assertIn("please write '50%%' to pass in '50%'", msg) + + # Test non-Windows behavior + sys.platform = "linux" + with self.assertRaises(typer.BadParameter) as cm: + parse_percentage("50") + msg = str(cm.exception) + self.assertIn("Expected percentage like 50% but got '50'", msg) + self.assertNotIn("please write '50%%' to pass in '50%'", msg) + finally: + sys.platform = orig_platform + + def test_parse_invalid_percentage_non_numeric(self): + with self.assertRaises(typer.BadParameter) as cm: + parse_percentage("abc%") + msg = str(cm.exception) + self.assertIn("Expected percentage like 50% but got 'abc%'", msg) + + def test_percentage_class_methods(self): + pct = Percentage(0.75) + self.assertEqual(str(pct), "75.0%") + self.assertEqual(float(pct), 0.75) + + +class DurationTest(TestCase): + def test_convert_to_seconds(self): + self.assertEqual(convert_to_seconds('30s'), 30) + self.assertEqual(convert_to_seconds('5m'), 300) + self.assertEqual(convert_to_seconds('1h30m'), 5400) + self.assertEqual(convert_to_seconds('1d10h15m'), 123300) + self.assertEqual(convert_to_seconds('15m 1d 10h'), 123300) + self.assertEqual(convert_to_seconds('1w'), 604800) # 7 days + + # Test numeric only + self.assertEqual(convert_to_seconds('3600'), 3600) + + def test_convert_to_seconds_invalid(self): + with self.assertRaises(ValueError): + convert_to_seconds('1h30k') + + def test_parse_duration(self): + duration = parse_duration("30s") + self.assertIsInstance(duration, Duration) + self.assertEqual(duration.seconds, 30) + self.assertEqual(float(duration), 30) + self.assertEqual(str(duration), "30.0s") + + def test_parse_duration_invalid(self): + # Note: convert_to_seconds returns 0.0 for invalid input instead of raising ValueError + # So parse_duration returns Duration(0.0) for invalid input + duration = parse_duration("invalid") + self.assertEqual(duration.seconds, 0.0) + + +class KeyValueTest(TestCase): + def test_parse_key_value_equals(self): + kv = parse_key_value("key=value") + self.assertIsInstance(kv, KeyValue) + self.assertEqual(kv.key, "key") + self.assertEqual(kv.value, "value") + self.assertEqual(str(kv), "key=value") + + # Test tuple-like behavior + self.assertEqual(kv[0], "key") + self.assertEqual(kv[1], "value") + self.assertEqual(list(kv), ["key", "value"]) + + def test_parse_key_value_colon(self): + kv = parse_key_value("key:value") + self.assertEqual(kv.key, "key") + self.assertEqual(kv.value, "value") + + def test_parse_key_value_with_spaces(self): + kv = parse_key_value(" key = value ") + self.assertEqual(kv.key, "key") + self.assertEqual(kv.value, "value") + + def test_parse_key_value_with_multiple_delimiters(self): + # Should split on first occurrence only + kv = parse_key_value("key=value=extra") + self.assertEqual(kv.key, "key") + self.assertEqual(kv.value, "value=extra") + + def test_parse_key_value_invalid(self): + with self.assertRaises(typer.BadParameter) as cm: + parse_key_value("invalid") + msg = str(cm.exception) + self.assertIn("Expected a key-value pair formatted as --option key=value, but got 'invalid'", msg) + + def test_validate_key_value_compat(self): + # Test backward compatibility function + result = validate_key_value("key=value") + self.assertEqual(result, ("key", "value")) + + def test_key_value_compat_function(self): + # Test the KEY_VALUE constant + result = KEY_VALUE("key=value") + self.assertEqual(result, ("key", "value")) + + +class FractionTest(TestCase): + def test_parse_fraction(self): + frac = parse_fraction("3/4") + self.assertIsInstance(frac, Fraction) + self.assertEqual(frac.numerator, 3) + self.assertEqual(frac.denominator, 4) + self.assertEqual(str(frac), "3/4") + self.assertEqual(float(frac), 0.75) + + # Test tuple-like behavior + self.assertEqual(frac[0], 3) + self.assertEqual(frac[1], 4) + self.assertEqual(list(frac), [3, 4]) + + def test_parse_fraction_with_spaces(self): + frac = parse_fraction(" 1 / 2 ") + self.assertEqual(frac.numerator, 1) + self.assertEqual(frac.denominator, 2) + + def test_parse_fraction_invalid(self): + with self.assertRaises(typer.BadParameter) as cm: + parse_fraction("invalid") + msg = str(cm.exception) + self.assertIn("Expected fraction like 1/2 but got 'invalid'", msg) + + def test_parse_fraction_invalid_numbers(self): + with self.assertRaises(typer.BadParameter): + parse_fraction("a/b") + + +class DateTimeWithTimezoneTest(TestCase): + def test_parse_datetime_with_timezone(self): + dt_str = "2023-10-01 12:00:00+00:00" + dt_obj = parse_datetime_with_timezone(dt_str) + self.assertIsInstance(dt_obj, DateTimeWithTimezone) + self.assertEqual(dt_obj.dt.year, 2023) + self.assertEqual(dt_obj.dt.month, 10) + self.assertEqual(dt_obj.dt.day, 1) + self.assertEqual(dt_obj.dt.hour, 12) + # dateutil.parser creates tzutc() which is equivalent to but not equal to timezone.utc + self.assertEqual(dt_obj.dt.utcoffset(), timezone.utc.utcoffset(None)) + + def test_parse_datetime_without_timezone(self): + dt_str = "2023-10-01 12:00:00" + dt_obj = parse_datetime_with_timezone(dt_str) + self.assertEqual(dt_obj.dt.tzinfo, tzlocal()) + + def test_parse_datetime_iso_format(self): + dt_str = "2023-10-01T20:00:00Z" + dt_obj = parse_datetime_with_timezone(dt_str) + # dateutil.parser creates tzutc() which is equivalent to but not equal to timezone.utc + self.assertEqual(dt_obj.dt.utcoffset(), timezone.utc.utcoffset(None)) + + def test_parse_datetime_invalid(self): + with self.assertRaises(typer.BadParameter) as cm: + parse_datetime_with_timezone("invalid") + msg = str(cm.exception) + self.assertIn("Expected datetime like 2023-10-01T12:00:00 but got 'invalid'", msg) + + def test_datetime_with_timezone_methods(self): + dt_obj = parse_datetime_with_timezone("2023-10-01T12:00:00Z") + self.assertEqual(dt_obj.datetime(), dt_obj.dt) + # Test string representation + self.assertIn("2023-10-01T12:00:00", str(dt_obj)) + + def test_validate_datetime_with_tz_compat(self): + # Test backward compatibility function + result = validate_datetime_with_tz("2023-10-01T12:00:00Z") + self.assertIsInstance(result, datetime.datetime) + # dateutil.parser creates tzutc() which is equivalent to but not equal to timezone.utc + self.assertEqual(result.utcoffset(), timezone.utc.utcoffset(None)) + + def test_datetime_with_tz_compat_function(self): + # Test the DATETIME_WITH_TZ constant + result = DATETIME_WITH_TZ("2023-10-01T12:00:00Z") + self.assertIsInstance(result, datetime.datetime) + + def test_validate_past_datetime(self): + # Test with None + self.assertIsNone(validate_past_datetime(None)) + + # Test with past datetime + past_dt = datetime.datetime(2020, 1, 1, tzinfo=tzlocal()) + self.assertEqual(validate_past_datetime(past_dt), past_dt) + + # Test with future datetime + future_dt = datetime.datetime(2030, 1, 1, tzinfo=tzlocal()) + with self.assertRaises(typer.BadParameter) as cm: + validate_past_datetime(future_dt) + msg = str(cm.exception) + self.assertIn("The provided timestamp must be in the past", msg) + + # Test with non-datetime object + with self.assertRaises(typer.BadParameter) as cm: + validate_past_datetime("not a datetime") + msg = str(cm.exception) + self.assertIn("Expected a datetime object", msg) + + +class EmojiTest(TestCase): + def test_emoji_function(self): + # Test with fallback + result = emoji("🎉", "!") + self.assertIn(result, ["🎉", "!"]) # Depends on system capability + + # Test without fallback + result = emoji("🎉") + self.assertIn(result, ["🎉", ""]) # Depends on system capability + + def test_emoji_constant(self): + # EMOJI should be a boolean + self.assertIsInstance(EMOJI, bool)