diff --git a/launchable/commands/record/build.py b/launchable/commands/record/build.py index 8f5d6013d..ddaec2547 100644 --- a/launchable/commands/record/build.py +++ b/launchable/commands/record/build.py @@ -13,6 +13,8 @@ from ...utils import subprocess from ...utils.authentication import get_org_workspace from ...utils.click import DATETIME_WITH_TZ, KEY_VALUE, validate_past_datetime +from ...utils.commands import Command +from ...utils.fail_fast_mode import set_fail_fast_mode, warn_and_exit_if_fail_fast_mode from ...utils.launchable_client import LaunchableClient from ...utils.session import clean_session_files, write_build from .commit import commit @@ -113,6 +115,10 @@ def build( links: Sequence[Tuple[str, str]], branches: Sequence[str], lineage: str, timestamp: Optional[datetime.datetime]): + tracking_client = TrackingClient(Command.RECORD_BUILD, app=ctx.obj) + client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client) + set_fail_fast_mode(client.is_fail_fast_mode()) + if "/" in build_name or "%2f" in build_name.lower(): sys.exit("--name must not contain a slash and an encoded slash") if "%25" in build_name: @@ -266,12 +272,7 @@ def compute_hash_and_branch(ws: List[Workspace]): sys.exit(1) if not ws_by_name.get(kv[0]): - click.echo(click.style( - "Invalid repository name {} in a --branch option. ".format(kv[0]), - fg="yellow"), - err=True) - # TODO: is there any reason this is not an error? for now erring on caution - # sys.exit(1) + warn_and_exit_if_fail_fast_mode("Invalid repository name {repo} in a --branch option.\nThe repository “{repo}” is not specified via `--source` or `--commit` option.".format(repo=kv[0])) # noqa: E501 branch_name_map[kv[0]] = kv[1] @@ -324,8 +325,6 @@ def compute_links(): }) return _links - tracking_client = TrackingClient(Tracking.Command.RECORD_BUILD, app=ctx.obj) - client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client) try: payload = { "buildNumber": build_name, diff --git a/launchable/commands/record/commit.py b/launchable/commands/record/commit.py index 36775a2e9..5cd831ae2 100644 --- a/launchable/commands/record/commit.py +++ b/launchable/commands/record/commit.py @@ -10,8 +10,10 @@ from launchable.utils.tracking import Tracking, TrackingClient from ...app import Application +from ...utils.commands import Command from ...utils.commit_ingester import upload_commits from ...utils.env_keys import COMMIT_TIMEOUT, REPORT_ERROR_KEY +from ...utils.fail_fast_mode import set_fail_fast_mode, warn_and_exit_if_fail_fast_mode from ...utils.git_log_parser import parse_git_log from ...utils.http_client import get_base_url from ...utils.java import cygpath, get_java_command @@ -53,13 +55,14 @@ def commit(ctx, source: str, executable: bool, max_days: int, scrub_pii: bool, i if executable == 'docker': sys.exit("--executable docker is no longer supported") + tracking_client = TrackingClient(Command.COMMIT, app=ctx.obj) + client = LaunchableClient(tracking_client=tracking_client, app=ctx.obj) + set_fail_fast_mode(client.is_fail_fast_mode()) + if import_git_log_output: _import_git_log(import_git_log_output, ctx.obj) return - tracking_client = TrackingClient(Tracking.Command.COMMIT, app=ctx.obj) - client = LaunchableClient(tracking_client=tracking_client, app=ctx.obj) - # Commit messages are not collected in the default. is_collect_message = False try: @@ -81,13 +84,9 @@ def commit(ctx, source: str, executable: bool, max_days: int, scrub_pii: bool, i if os.getenv(REPORT_ERROR_KEY): raise e else: - click.echo(click.style( + warn_and_exit_if_fail_fast_mode( "Couldn't get commit history from `{}`. Do you run command root of git-controlled directory? " - "If not, please set a directory use by --source option." - .format(cwd), - fg='yellow'), - err=True) - print(e) + "If not, please set a directory use by --source option.\nerror: {}".format(cwd, e)) def exec_jar(source: str, max_days: int, app: Application, is_collect_message: bool): @@ -141,10 +140,7 @@ def _import_git_log(output_file: str, app: Application): if os.getenv(REPORT_ERROR_KEY): raise e else: - click.echo( - click.style("Failed to import the git-log output", fg='yellow'), - err=True) - print(e) + warn_and_exit_if_fail_fast_mode("Failed to import the git-log output\n error: {}".format(e)) def _build_proxy_option(https_proxy: Optional[str]) -> List[str]: diff --git a/launchable/commands/record/session.py b/launchable/commands/record/session.py index a2bf8f29f..b5376e1a0 100644 --- a/launchable/commands/record/session.py +++ b/launchable/commands/record/session.py @@ -12,6 +12,8 @@ from launchable.utils.tracking import Tracking, TrackingClient from ...utils.click import KEY_VALUE +from ...utils.commands import Command +from ...utils.fail_fast_mode import FailFastModeValidateParams, fail_fast_mode_validate, set_fail_fast_mode from ...utils.launchable_client import LaunchableClient from ...utils.no_build import NO_BUILD_BUILD_NAME from ...utils.session import _session_file_path, read_build, write_session @@ -132,6 +134,17 @@ def session( you should set print_session = False because users don't expect to print session ID to the subset output. """ + tracking_client = TrackingClient(Command.RECORD_SESSION, app=ctx.obj) + client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client) + set_fail_fast_mode(client.is_fail_fast_mode()) + + fail_fast_mode_validate(FailFastModeValidateParams( + command=Command.RECORD_SESSION, + build=build_name, + is_no_build=is_no_build, + test_suite=test_suite, + )) + if not is_no_build and not build_name: raise click.UsageError("Error: Missing option '--build'") @@ -143,9 +156,6 @@ def session( build_name = NO_BUILD_BUILD_NAME - tracking_client = TrackingClient(Tracking.Command.RECORD_SESSION, app=ctx.obj) - client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client) - if session_name: sub_path = "builds/{}/test_session_names/{}".format(build_name, session_name) try: diff --git a/launchable/commands/record/tests.py b/launchable/commands/record/tests.py index 39b0a396d..8d07a075e 100644 --- a/launchable/commands/record/tests.py +++ b/launchable/commands/record/tests.py @@ -17,7 +17,10 @@ from ...testpath import FilePathNormalizer, TestPathComponent, unparse_test_path from ...utils.click import DATETIME_WITH_TZ, KEY_VALUE, validate_past_datetime +from ...utils.commands import Command from ...utils.exceptions import InvalidJUnitXMLException +from ...utils.fail_fast_mode import (FailFastModeValidateParams, fail_fast_mode_validate, + set_fail_fast_mode, warn_and_exit_if_fail_fast_mode) from ...utils.launchable_client import LaunchableClient from ...utils.logger import Logger from ...utils.no_build import NO_BUILD_BUILD_NAME, NO_BUILD_TEST_SESSION_ID @@ -193,8 +196,19 @@ def tests( test_runner = context.invoked_subcommand - tracking_client = TrackingClient(Tracking.Command.RECORD_TESTS, app=context.obj) + tracking_client = TrackingClient(Command.RECORD_TESTS, app=context.obj) client = LaunchableClient(test_runner=test_runner, app=context.obj, tracking_client=tracking_client) + set_fail_fast_mode(client.is_fail_fast_mode()) + + fail_fast_mode_validate(FailFastModeValidateParams( + command=Command.RECORD_TESTS, + session=session, + build=build_name, + flavor=flavor, + links=links, + is_no_build=is_no_build, + test_suite=test_suite, + )) file_path_normalizer = FilePathNormalizer(base_path, no_base_path_inference=no_base_path_inference) @@ -209,11 +223,10 @@ def tests( raise click.UsageError(message=msg) # noqa: E501 if is_no_build and session: - click.echo( - click.style( - "WARNING: `--session` and `--no-build` are set.\nUsing --session option value ({}) and ignoring `--no-build` option".format(session), # noqa: E501 - fg='yellow'), - err=True) + warn_and_exit_if_fail_fast_mode( + "WARNING: `--session` and `--no-build` are set.\nUsing --session option value ({}) and ignoring `--no-build` option".format(session), # noqa: E501 + ) + is_no_build = False try: @@ -355,11 +368,12 @@ def parse(report: str) -> Generator[CaseEventType, None, None]: try: xml = JUnitXml.fromfile(report, f) except Exception as e: - click.echo(click.style("Warning: error reading JUnitXml file {filename}: {error}".format( - filename=report, error=e), fg="yellow"), err=True) # `JUnitXml.fromfile()` will raise `JUnitXmlError` and other lxml related errors # if the file has wrong format. # https://github.com/weiwei/junitparser/blob/master/junitparser/junitparser.py#L321 + warn_and_exit_if_fail_fast_mode( + "Warning: error reading JUnitXml file {filename}: {error}".format( + filename=report, error=e)) return if isinstance(xml, JUnitXml): testsuites = [suite for suite in xml] @@ -373,8 +387,9 @@ def parse(report: str) -> Generator[CaseEventType, None, None]: for case in suite: yield CaseEvent.from_case_and_suite(self.path_builder, case, suite, report, self.metadata_builder) except Exception as e: - click.echo(click.style("Warning: error parsing JUnitXml file {filename}: {error}".format( - filename=report, error=e), fg="yellow"), err=True) + warn_and_exit_if_fail_fast_mode( + "Warning: error parsing JUnitXml file {filename}: {error}".format( + filename=report, error=e)) self.parse_func = parse @@ -504,25 +519,12 @@ def send(payload: Dict[str, Union[str, List]]) -> None: if res.status_code == HTTPStatus.NOT_FOUND: if session: build, _ = parse_session(session) - click.echo( - click.style( - "Session {} was not found. " - "Make sure to run `launchable record session --build {}` " - "before `launchable record tests`".format( - session, - build), - 'yellow'), - err=True) + warn_and_exit_if_fail_fast_mode( + "Session {} was not found. Make sure to run `launchable record session --build {}` before `launchable record tests`".format(session, build)) # noqa: E501 + elif build_name: - click.echo( - click.style( - "Build {} was not found. " - "Make sure to run `launchable record build --name {}` " - "before `launchable record tests`".format( - build_name, - build_name), - 'yellow'), - err=True) + warn_and_exit_if_fail_fast_mode( + "Build {} was not found. Make sure to run `launchable record build --name {}` before `launchable record tests`".format(build_name, build_name)) # noqa: E501 res.raise_for_status() @@ -606,18 +608,15 @@ def recorded_result() -> Tuple[int, int, int, float]: if count == 0: if len(self.skipped_reports) != 0: - click.echo(click.style( + warn_and_exit_if_fail_fast_mode( "{} test report(s) were skipped because they were created before this build was recorded.\n" "Make sure to run your tests after you run `launchable record build`.\n" - "Otherwise, if these are really correct test reports, use the `--allow-test-before-build` option.".format( - len(self.skipped_reports)), 'yellow')) + "Otherwise, if these are really correct test reports, use the `--allow-test-before-build` option.". + format(len(self.skipped_reports))) return else: - click.echo( - click.style( - "Looks like tests didn't run? " - "If not, make sure the right files/directories were passed into `launchable record tests`", - 'yellow')) + warn_and_exit_if_fail_fast_mode( + "Looks like tests didn't run? If not, make sure the right files/directories were passed into `launchable record tests`") # noqa: E501 return file_count = len(self.reports) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index a62e5e686..2426ee165 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -17,7 +17,10 @@ from ..app import Application from ..testpath import FilePathNormalizer, TestPath from ..utils.click import DURATION, KEY_VALUE, PERCENTAGE, DurationType, PercentageType, ignorable_error +from ..utils.commands import Command from ..utils.env_keys import REPORT_ERROR_KEY +from ..utils.fail_fast_mode import (FailFastModeValidateParams, fail_fast_mode_validate, + set_fail_fast_mode, warn_and_exit_if_fail_fast_mode) from ..utils.launchable_client import LaunchableClient from .helper import find_or_create_session from .test_path_writer import TestPathWriter @@ -225,7 +228,23 @@ def subset( test_suite: Optional[str] = None, ): app = context.obj - tracking_client = TrackingClient(Tracking.Command.SUBSET, app=app) + tracking_client = TrackingClient(Command.SUBSET, app=app) + client = LaunchableClient( + test_runner=context.invoked_subcommand, + app=app, + tracking_client=tracking_client) + + set_fail_fast_mode(client.is_fail_fast_mode()) + fail_fast_mode_validate(FailFastModeValidateParams( + command=Command.SUBSET, + session=session, + build=build_name, + flavor=flavor, + is_observation=is_observation, + links=links, + is_no_build=is_no_build, + test_suite=test_suite, + )) if is_observation and is_output_exclusion_rules: msg = ( @@ -262,15 +281,12 @@ def subset( sys.exit(1) if is_no_build and session: - click.echo( - click.style( - "WARNING: `--session` and `--no-build` are set.\nUsing --session option value ({}) and ignoring `--no-build` option".format(session), # noqa: E501 - fg='yellow'), - err=True) + warn_and_exit_if_fail_fast_mode( + "WARNING: `--session` and `--no-build` are set.\nUsing --session option value ({}) and ignoring `--no-build` option".format(session)) # noqa: E501 is_no_build = False session_id = None - tracking_client = TrackingClient(Tracking.Command.SUBSET, app=app) + try: if session_name: if not build_name: @@ -410,12 +426,10 @@ def stdin(self) -> Union[TextIO, List]: they didn't feed anything from stdin """ if sys.stdin.isatty(): - click.echo( - click.style( - "Warning: this command reads from stdin but it doesn't appear to be connected to anything. " - "Did you forget to pipe from another command?", - fg='yellow'), - err=True) + warn_and_exit_if_fail_fast_mode( + "Warning: this command reads from stdin but it doesn't appear to be connected to anything. " + "Did you forget to pipe from another command?" + ) return sys.stdin @staticmethod @@ -537,10 +551,6 @@ def run(self): else: try: test_runner = context.invoked_subcommand - client = LaunchableClient( - test_runner=test_runner, - app=app, - tracking_client=tracking_client) # temporarily extend the timeout because subset API response has become slow # TODO: remove this line when API response return respose @@ -589,7 +599,7 @@ def run(self): e, "Warning: the service failed to subset. Falling back to running all tests") if len(original_subset) == 0: - click.echo(click.style("Error: no tests found matching the path.", 'yellow'), err=True) + warn_and_exit_if_fail_fast_mode("Error: no tests found matching the path.") return if split: diff --git a/launchable/commands/verify.py b/launchable/commands/verify.py index 95db8c567..3af7e37f3 100644 --- a/launchable/commands/verify.py +++ b/launchable/commands/verify.py @@ -12,6 +12,7 @@ from ..utils.authentication import get_org_workspace from ..utils.click import emoji +from ..utils.commands import Command from ..utils.java import get_java_command from ..utils.launchable_client import LaunchableClient from ..version import __version__ as version @@ -61,7 +62,7 @@ def verify(context: click.core.Context): # Click gracefully. org, workspace = get_org_workspace() - tracking_client = TrackingClient(Tracking.Command.VERIFY, app=context.obj) + tracking_client = TrackingClient(Command.VERIFY, app=context.obj) client = LaunchableClient(tracking_client=tracking_client, app=context.obj) java = get_java_command() diff --git a/launchable/utils/commands.py b/launchable/utils/commands.py new file mode 100644 index 000000000..6a8e357b6 --- /dev/null +++ b/launchable/utils/commands.py @@ -0,0 +1,13 @@ +from enum import Enum + + +class Command(Enum): + VERIFY = 'VERIFY' + RECORD_TESTS = 'RECORD_TESTS' + RECORD_BUILD = 'RECORD_BUILD' + RECORD_SESSION = 'RECORD_SESSION' + SUBSET = 'SUBSET' + COMMIT = 'COMMIT' + + def display_name(self): + return self.value.lower().replace('_', ' ') diff --git a/launchable/utils/fail_fast_mode.py b/launchable/utils/fail_fast_mode.py new file mode 100644 index 000000000..e24d580ee --- /dev/null +++ b/launchable/utils/fail_fast_mode.py @@ -0,0 +1,101 @@ +import sys +from typing import List, Optional, Sequence, Tuple + +import click + +from .commands import Command + +_fail_fast_mode_cache: Optional[bool] = None + + +def set_fail_fast_mode(enabled: bool): + global _fail_fast_mode_cache + _fail_fast_mode_cache = enabled + + +def is_fail_fast_mode() -> bool: + if _fail_fast_mode_cache: + return _fail_fast_mode_cache + + # Default to False if not set + return False + + +def warn_and_exit_if_fail_fast_mode(message: str): + color = 'red' if is_fail_fast_mode() else 'yellow' + message = click.style(message, fg=color) + + click.echo(message, err=True) + if is_fail_fast_mode(): + sys.exit(1) + + +class FailFastModeValidateParams: + def __init__(self, command: Command, build: Optional[str] = None, is_no_build: bool = False, + test_suite: Optional[str] = None, session: Optional[str] = None, + links: Sequence[Tuple[str, str]] = (), is_observation: bool = False, + flavor: Sequence[Tuple[str, str]] = ()): + self.command = command + self.build = build + self.is_no_build = is_no_build + self.test_suite = test_suite + self.session = session + self.links = links + self.is_observation = is_observation + self.flavor = flavor + + +def fail_fast_mode_validate(params: FailFastModeValidateParams): + if not is_fail_fast_mode(): + return + + if params.command == Command.RECORD_SESSION: + _validate_record_session(params) + if params.command == Command.SUBSET: + _validate_subset(params) + if params.command == Command.RECORD_TESTS: + _validate_record_tests(params) + + +def _validate_require_session_option(params: FailFastModeValidateParams) -> List[str]: + errors: List[str] = [] + cmd_name = params.command.display_name() + if params.session: + if params.test_suite: + errors.append("`--test-suite` option was ignored in the {} command. Add `--test-suite` option to the `record session` command instead.".format(cmd_name)) # noqa: E501 + + if params.is_observation: + errors.append( + "`--observation` was ignored in the {} command. Add `--observation` option to the `record session` command instead.".format(cmd_name)) # noqa: E501 + + if len(params.flavor) > 0: + errors.append( + "`--flavor` option was ignored in the {} command. Add `--flavor` option to the `record session` command instead.".format(cmd_name)) # noqa: E501 + + if len(params.links) > 0: + errors.append( + "`--link` option was ignored in the {} command. Add `link` option to the `record session` command instead.".format(cmd_name)) # noqa: E501 + + return errors + + +def _validate_record_session(params: FailFastModeValidateParams): + # Now, there isn't any validation for the `record session` command in fail-fast mode. + return + + +def _validate_subset(params: FailFastModeValidateParams): + errors = _validate_require_session_option(params) + _exit_if_errors(errors) + + +def _validate_record_tests(params: FailFastModeValidateParams): + errors = _validate_require_session_option(params) + _exit_if_errors(errors) + + +def _exit_if_errors(errors: List[str]): + if errors: + msg = "\n".join(map(lambda x: click.style(x, fg='red'), errors)) + click.echo(msg, err=True) + sys.exit(1) diff --git a/launchable/utils/launchable_client.py b/launchable/utils/launchable_client.py index e8774a80f..0f5347814 100644 --- a/launchable/utils/launchable_client.py +++ b/launchable/utils/launchable_client.py @@ -30,6 +30,7 @@ def __init__(self, tracking_client: Optional[TrackingClient] = None, base_url: s "Confirm that you set LAUNCHABLE_TOKEN " "(or LAUNCHABLE_ORGANIZATION and LAUNCHABLE_WORKSPACE) environment variable(s)\n" "See https://docs.launchableinc.com/getting-started#setting-your-api-key") + self._workspace_state_cache: dict = {} def request( self, @@ -99,3 +100,17 @@ def print_exception_and_recover(self, e: Exception, warning: Optional[str] = Non if warning: click.echo(click.style(warning, fg=warning_color), err=True) + + def is_fail_fast_mode(self) -> bool: + if 'fail_fast_mode' in self._workspace_state_cache: + return self._workspace_state_cache['fail_fast_mode'] + # TODO: call api and set the result to cache + try: + res = self.request("get", "state") + if res.status_code == 200: + self._workspace_state_cache['fail_fast_mode'] = res.json().get('isFailFastMode', False) + return self._workspace_state_cache['fail_fast_mode'] + except Exception as e: + self.print_exception_and_recover(e, "Failed to check fail-fast mode status") + + return False diff --git a/launchable/utils/tracking.py b/launchable/utils/tracking.py index cc22c5258..8358b23eb 100644 --- a/launchable/utils/tracking.py +++ b/launchable/utils/tracking.py @@ -8,6 +8,8 @@ from launchable.utils.http_client import _HttpClient, _join_paths from launchable.version import __version__ +from .commands import Command + class Tracking: # General events @@ -27,17 +29,9 @@ class ErrorEvent(Enum): INTERNAL_SERVER_ERROR = 'INTERNAL_SERVER_ERROR' UNEXPECTED_HTTP_STATUS_ERROR = 'UNEXPECTED_HTTP_STATUS_ERROR' - class Command(Enum): - VERIFY = 'VERIFY' - RECORD_TESTS = 'RECORD_TESTS' - RECORD_BUILD = 'RECORD_BUILD' - RECORD_SESSION = 'RECORD_SESSION' - SUBSET = 'SUBSET' - COMMIT = 'COMMIT' - class TrackingClient: - def __init__(self, command: Tracking.Command, base_url: str = "", session: Optional[Session] = None, + def __init__(self, command: Command, base_url: str = "", session: Optional[Session] = None, test_runner: Optional[str] = "", app: Optional[Application] = None): self.http_client = _HttpClient( base_url=base_url, diff --git a/tests/cli_test_case.py b/tests/cli_test_case.py index 9fe24536a..e64e0f57d 100644 --- a/tests/cli_test_case.py +++ b/tests/cli_test_case.py @@ -186,6 +186,14 @@ def setUp(self): self.workspace), json={'commitMessage': True}, status=200) + responses.add( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/state".format( + get_base_url(), + self.organization, + self.workspace), + json={'isFailFastMode': False}, + status=200) def get_test_files_dir(self): file_name = Path(inspect.getfile(self.__class__)) # obtain the file of the concrete type diff --git a/tests/commands/record/test_build.py b/tests/commands/record/test_build.py index bf1a7a4d7..5d6940867 100644 --- a/tests/commands/record/test_build.py +++ b/tests/commands/record/test_build.py @@ -37,7 +37,7 @@ def test_submodule(self, mock_check_output): # Name & Path should both reflect the submodule path self.assertTrue("| ./bar-zot | ./bar-zot | 8bccab48338219e73c3118ad71c8c98fbd32a4be |" in result.stdout, result.stdout) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal( { "buildNumber": "123", @@ -82,7 +82,7 @@ def test_no_submodule(self, mock_check_output): result = self.cli("record", "build", "--no-commit-collection", "--no-submodules", "--name", self.build_name) self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal( { "buildNumber": "123", @@ -114,7 +114,7 @@ def test_no_git_directory(self): self.cli("record", "build", "--no-commit-collection", "--commit", ".=c50f5de0f06fe16afa4fd1dd615e4903e40b42a2", "--name", self.build_name) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal( { "buildNumber": "123", @@ -144,7 +144,7 @@ def test_commit_option_and_build_option(self): result = self.cli("record", "build", "--no-commit-collection", "--commit", "A=abc12", "--name", self.build_name) self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal( { "buildNumber": "123", @@ -174,7 +174,7 @@ def test_commit_option_and_build_option(self): self.build_name) self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal( { "buildNumber": "123", @@ -204,7 +204,7 @@ def test_commit_option_and_build_option(self): self.build_name) self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal( { "buildNumber": "123", @@ -220,7 +220,7 @@ def test_commit_option_and_build_option(self): "timestamp": None }, payload) responses.calls.reset() - self.assertIn("Invalid repository name B in a --branch option. ", result.output) + self.assertIn("Invalid repository name B in a --branch option.", result.output) # case multiple --commit options and multiple --branch options result = self.cli( @@ -239,7 +239,7 @@ def test_commit_option_and_build_option(self): self.build_name) self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal( { "buildNumber": "123", @@ -261,6 +261,8 @@ def test_commit_option_and_build_option(self): }, payload) responses.calls.reset() + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_build_name_validation(self): result = self.cli("record", "build", "--no-commit-collection", "--name", "foo/hoge") self.assert_exit_code(result, 1) @@ -289,7 +291,7 @@ def test_with_timestamp(self, mock_check_output): "2025-01-23 12:34:56Z") self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal( { "buildNumber": "123", diff --git a/tests/commands/record/test_session.py b/tests/commands/record/test_session.py index 014e4a124..ddac44e15 100644 --- a/tests/commands/record/test_session.py +++ b/tests/commands/record/test_session.py @@ -26,7 +26,7 @@ def test_run_session_without_flavor(self): result = self.cli("record", "session", "--build", self.build_name) self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal({ "flavors": {}, "isObservation": False, @@ -47,7 +47,7 @@ def test_run_session_with_flavor(self): "--flavor", "key=value", "--flavor", "k:v", "--flavor", "k e y = v a l u e") self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal({ "flavors": { "key": "value", @@ -75,7 +75,7 @@ def test_run_session_with_observation(self): result = self.cli("record", "session", "--build", self.build_name, "--observation") self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal({ "flavors": {}, @@ -115,7 +115,7 @@ def test_run_session_with_session_name(self): result = self.cli("record", "session", "--build", self.build_name, "--session-name", self.session_name) self.assert_success(result) - payload = json.loads(responses.calls[3].request.body.decode()) + payload = json.loads(responses.calls[5].request.body.decode()) self.assert_json_orderless_equal({ "flavors": {}, "isObservation": False, @@ -136,7 +136,7 @@ def test_run_session_with_lineage(self): "--lineage", "example-lineage") self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal({ "flavors": {}, "isObservation": False, @@ -157,7 +157,7 @@ def test_run_session_with_test_suite(self): "--test-suite", "example-test-suite") self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal({ "flavors": {}, "isObservation": False, @@ -178,7 +178,7 @@ def test_run_session_with_timestamp(self): "--timestamp", "2023-10-01T12:00:00Z") self.assert_success(result) - payload = json.loads(responses.calls[0].request.body.decode()) + payload = json.loads(responses.calls[1].request.body.decode()) self.assert_json_orderless_equal({ "flavors": {}, "isObservation": False, diff --git a/tests/commands/test_api_error.py b/tests/commands/test_api_error.py index 1fd8a55ad..97fdff4ee 100644 --- a/tests/commands/test_api_error.py +++ b/tests/commands/test_api_error.py @@ -131,7 +131,7 @@ def test_record_build(self): self.assert_success(result) self.assertEqual(result.exception, None) # Since HTTPError is occurred outside of LaunchableClient, the count is 1. - self.assert_tracking_count(tracking=tracking, count=1) + self.assert_tracking_count(tracking=tracking, count=3) success_server.shutdown() thread.join() @@ -165,7 +165,7 @@ def test_record_build(self): self.assert_success(result) self.assertEqual(result.exception, None) # Since HTTPError is occurred outside of LaunchableClient, the count is 1. - self.assert_tracking_count(tracking=tracking, count=1) + self.assert_tracking_count(tracking=tracking, count=3) error_server.shutdown() thread.join() @@ -390,6 +390,14 @@ def test_all_workflow_when_server_down(self): CLI_TRACKING_URL.format( base=get_base_url()), body=ReadTimeout("error")) + # setup state + responses.replace( + responses.GET, + "{base}/intake/organizations/{org}/workspaces/{ws}/state".format( + base=get_base_url(), + org=self.organization, + ws=self.workspace), + body=ReadTimeout("error")) # setup build responses.replace( responses.POST, @@ -429,7 +437,7 @@ def test_all_workflow_when_server_down(self): self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=4) + self.assert_tracking_count(tracking=tracking, count=6) # set delete=False to solve the error `PermissionError: [Errno 13] Permission denied:` on Windows. with tempfile.NamedTemporaryFile(delete=False) as rest_file: @@ -446,12 +454,12 @@ def test_all_workflow_when_server_down(self): self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=6) + self.assert_tracking_count(tracking=tracking, count=9) result = self.cli("record", "tests", "--session", self.session, "minitest", str(self.test_files_dir) + "/") self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=9) + self.assert_tracking_count(tracking=tracking, count=13) def assert_tracking_count(self, tracking, count: int): # Prior to 3.6, `Response` object can't be obtained. diff --git a/tests/commands/test_helper.py b/tests/commands/test_helper.py index 3a3924e69..82a05c22f 100644 --- a/tests/commands/test_helper.py +++ b/tests/commands/test_helper.py @@ -6,8 +6,9 @@ import responses # type: ignore from launchable.commands.helper import _check_observation_mode_status +from launchable.utils.commands import Command from launchable.utils.http_client import get_base_url -from launchable.utils.tracking import Tracking, TrackingClient +from launchable.utils.tracking import TrackingClient from tests.cli_test_case import CliTestCase @@ -21,7 +22,7 @@ def test_check_observation_mode_status(self): self.session_id, ) - tracking_client = TrackingClient(Tracking.Command.RECORD_TESTS) + tracking_client = TrackingClient(Command.RECORD_TESTS) with mock.patch('sys.stderr', new=StringIO()) as stderr: _check_observation_mode_status(test_session, False, tracking_client=tracking_client) diff --git a/tests/commands/test_subset.py b/tests/commands/test_subset.py index 076ca7e49..247ec2c67 100644 --- a/tests/commands/test_subset.py +++ b/tests/commands/test_subset.py @@ -205,7 +205,7 @@ def test_subset_targetless(self): mix_stderr=False) self.assert_success(result) - payload = json.loads(gzip.decompress(responses.calls[0].request.body).decode()) + payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) self.assertTrue(payload.get('useServerSideOptimizationTarget')) @responses.activate @@ -238,7 +238,7 @@ def test_subset_goalspec(self): input="test_aaa.py") self.assert_success(result) - payload = json.loads(gzip.decompress(responses.calls[0].request.body).decode()) + payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) self.assertEqual(payload.get('goal').get('goal'), "foo(),bar(zot=3%)") @responses.activate @@ -280,7 +280,7 @@ def test_subset_ignore_flaky_tests_above(self): mix_stderr=False) self.assert_success(result) - payload = json.loads(gzip.decompress(responses.calls[0].request.body).decode()) + payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) self.assertEqual(payload.get('dropFlakinessThreshold'), 0.05) @responses.activate @@ -545,5 +545,5 @@ def test_subset_prioritize_tests_failed_within_hours(self): mix_stderr=False) self.assert_success(result) - payload = json.loads(gzip.decompress(responses.calls[0].request.body).decode()) + payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) self.assertEqual(payload.get('hoursToPrioritizeFailedTest'), 24) diff --git a/tests/utils/test_fail_fast_mode.py b/tests/utils/test_fail_fast_mode.py new file mode 100644 index 000000000..c150e5f8f --- /dev/null +++ b/tests/utils/test_fail_fast_mode.py @@ -0,0 +1,40 @@ +import io +from contextlib import contextmanager, redirect_stderr + +from launchable.utils.commands import Command +from launchable.utils.fail_fast_mode import FailFastModeValidateParams, fail_fast_mode_validate +from tests.cli_test_case import CliTestCase + + +class FailFastModeTest(CliTestCase): + def test_fail_fast_mode_validate(self): + params = FailFastModeValidateParams( + command=Command.SUBSET, + session='session1', + test_suite='test_suite1', + is_observation=True, + ) + stderr = io.StringIO() + + with self.assertRaises(SystemExit) as cm: + with tmp_set_fail_fast_mode(False), redirect_stderr(stderr): + # `--observation` option won't be applied but the cli won't be error + fail_fast_mode_validate(params) + self.assertEqual(stderr.getvalue(), "") + + with tmp_set_fail_fast_mode(True), redirect_stderr(stderr): + fail_fast_mode_validate(params) + self.assertEqual(cm.exception.code, 1) + self.assertIn("ignored", stderr.getvalue()) + + +@contextmanager +def tmp_set_fail_fast_mode(enabled: bool): + from launchable.utils.fail_fast_mode import _fail_fast_mode_cache, set_fail_fast_mode + original = _fail_fast_mode_cache + try: + set_fail_fast_mode(enabled) + yield + finally: + if original is not None: + set_fail_fast_mode(original)