Introduce fail fast mode

launchable/commands/record/build.py

@@ -13,6 +13,8 @@
 from ...utils import subprocess
 from ...utils.authentication import get_org_workspace
 from ...utils.click import DATETIME_WITH_TZ, KEY_VALUE, validate_past_datetime
+from ...utils.commands import Command
+from ...utils.fail_fast_mode import set_fail_fast_mode, warn_and_exit_if_fail_fast_mode
 from ...utils.launchable_client import LaunchableClient
 from ...utils.session import clean_session_files, write_build
 from .commit import commit
@@ -113,6 +115,10 @@ def build(
         links: Sequence[Tuple[str, str]],
         branches: Sequence[str], lineage: str, timestamp: Optional[datetime.datetime]):
 
+    tracking_client = TrackingClient(Command.RECORD_BUILD, app=ctx.obj)
+    client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client)
+    set_fail_fast_mode(client.is_fail_fast_mode())
+
     if "/" in build_name or "%2f" in build_name.lower():
         sys.exit("--name must not contain a slash and an encoded slash")
     if "%25" in build_name:
@@ -266,12 +272,7 @@ def compute_hash_and_branch(ws: List[Workspace]):
                     sys.exit(1)
 
                 if not ws_by_name.get(kv[0]):
-                    click.echo(click.style(
-                        "Invalid repository name {} in a --branch option. ".format(kv[0]),
-                        fg="yellow"),
-                        err=True)
-                    # TODO: is there any reason this is not an error? for now erring on caution
-                    # sys.exit(1)
+                    warn_and_exit_if_fail_fast_mode("Invalid repository name {repo} in a --branch option.\nThe repository “{repo}” is not specified via `--source` or `--commit` option.".format(repo=kv[0]))  # noqa: E501
 
                 branch_name_map[kv[0]] = kv[1]
 
@@ -324,8 +325,6 @@ def compute_links():
                 })
             return _links
 
-        tracking_client = TrackingClient(Tracking.Command.RECORD_BUILD, app=ctx.obj)
-        client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client)
         try:
             payload = {
                 "buildNumber": build_name,

launchable/commands/record/commit.py

@@ -10,8 +10,10 @@
 from launchable.utils.tracking import Tracking, TrackingClient
 
 from ...app import Application
+from ...utils.commands import Command
 from ...utils.commit_ingester import upload_commits
 from ...utils.env_keys import COMMIT_TIMEOUT, REPORT_ERROR_KEY
+from ...utils.fail_fast_mode import set_fail_fast_mode, warn_and_exit_if_fail_fast_mode
 from ...utils.git_log_parser import parse_git_log
 from ...utils.http_client import get_base_url
 from ...utils.java import cygpath, get_java_command
@@ -53,13 +55,14 @@ def commit(ctx, source: str, executable: bool, max_days: int, scrub_pii: bool, i
     if executable == 'docker':
         sys.exit("--executable docker is no longer supported")
 
+    tracking_client = TrackingClient(Command.COMMIT, app=ctx.obj)
+    client = LaunchableClient(tracking_client=tracking_client, app=ctx.obj)
+    set_fail_fast_mode(client.is_fail_fast_mode())
+
     if import_git_log_output:
         _import_git_log(import_git_log_output, ctx.obj)
         return
 
-    tracking_client = TrackingClient(Tracking.Command.COMMIT, app=ctx.obj)
-    client = LaunchableClient(tracking_client=tracking_client, app=ctx.obj)
-
     # Commit messages are not collected in the default.
     is_collect_message = False
     try:
@@ -81,13 +84,9 @@ def commit(ctx, source: str, executable: bool, max_days: int, scrub_pii: bool, i
         if os.getenv(REPORT_ERROR_KEY):
             raise e
         else:
-            click.echo(click.style(
+            warn_and_exit_if_fail_fast_mode(
                 "Couldn't get commit history from `{}`. Do you run command root of git-controlled directory? "
-                "If not, please set a directory use by --source option."
-                .format(cwd),
-                fg='yellow'),
-                err=True)
-            print(e)
+                "If not, please set a directory use by --source option.\nerror: {}".format(cwd, e))
 
 
 def exec_jar(source: str, max_days: int, app: Application, is_collect_message: bool):
@@ -141,10 +140,7 @@ def _import_git_log(output_file: str, app: Application):
         if os.getenv(REPORT_ERROR_KEY):
             raise e
         else:
-            click.echo(
-                click.style("Failed to import the git-log output", fg='yellow'),
-                err=True)
-            print(e)
+            warn_and_exit_if_fail_fast_mode("Failed to import the git-log output\n error: {}".format(e))
 
 
 def _build_proxy_option(https_proxy: Optional[str]) -> List[str]:

launchable/commands/record/session.py

@@ -12,6 +12,8 @@
 from launchable.utils.tracking import Tracking, TrackingClient
 
 from ...utils.click import KEY_VALUE
+from ...utils.commands import Command
+from ...utils.fail_fast_mode import FailFastModeValidateParams, fail_fast_mode_validate, set_fail_fast_mode
 from ...utils.launchable_client import LaunchableClient
 from ...utils.no_build import NO_BUILD_BUILD_NAME
 from ...utils.session import _session_file_path, read_build, write_session
@@ -132,6 +134,17 @@ def session(
     you should set print_session = False because users don't expect to print session ID to the subset output.
     """
 
+    tracking_client = TrackingClient(Command.RECORD_SESSION, app=ctx.obj)
+    client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client)
+    set_fail_fast_mode(client.is_fail_fast_mode())
+
+    fail_fast_mode_validate(FailFastModeValidateParams(
+        command=Command.RECORD_SESSION,
+        build=build_name,
+        is_no_build=is_no_build,
+        test_suite=test_suite,
+    ))
+
     if not is_no_build and not build_name:
         raise click.UsageError("Error: Missing option '--build'")
 
@@ -143,9 +156,6 @@ def session(
 
         build_name = NO_BUILD_BUILD_NAME
 
-    tracking_client = TrackingClient(Tracking.Command.RECORD_SESSION, app=ctx.obj)
-    client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client)
-
     if session_name:
         sub_path = "builds/{}/test_session_names/{}".format(build_name, session_name)
         try:

launchable/commands/record/tests.py

@@ -17,7 +17,10 @@
 
 from ...testpath import FilePathNormalizer, TestPathComponent, unparse_test_path
 from ...utils.click import DATETIME_WITH_TZ, KEY_VALUE, validate_past_datetime
+from ...utils.commands import Command
 from ...utils.exceptions import InvalidJUnitXMLException
+from ...utils.fail_fast_mode import (FailFastModeValidateParams, fail_fast_mode_validate,
+                                     set_fail_fast_mode, warn_and_exit_if_fail_fast_mode)
 from ...utils.launchable_client import LaunchableClient
 from ...utils.logger import Logger
 from ...utils.no_build import NO_BUILD_BUILD_NAME, NO_BUILD_TEST_SESSION_ID
@@ -193,8 +196,19 @@ def tests(
 
     test_runner = context.invoked_subcommand
 
-    tracking_client = TrackingClient(Tracking.Command.RECORD_TESTS, app=context.obj)
+    tracking_client = TrackingClient(Command.RECORD_TESTS, app=context.obj)
     client = LaunchableClient(test_runner=test_runner, app=context.obj, tracking_client=tracking_client)
+    set_fail_fast_mode(client.is_fail_fast_mode())
+
+    fail_fast_mode_validate(FailFastModeValidateParams(
+        command=Command.RECORD_TESTS,
+        session=session,
+        build=build_name,
+        flavor=flavor,
+        links=links,
+        is_no_build=is_no_build,
+        test_suite=test_suite,
+    ))
 
     file_path_normalizer = FilePathNormalizer(base_path, no_base_path_inference=no_base_path_inference)
 
@@ -209,11 +223,10 @@ def tests(
         raise click.UsageError(message=msg)  # noqa: E501
 
     if is_no_build and session:
-        click.echo(
-            click.style(
-                "WARNING: `--session` and `--no-build` are set.\nUsing --session option value ({}) and ignoring `--no-build` option".format(session),  # noqa: E501
-                fg='yellow'),
-            err=True)
+        warn_and_exit_if_fail_fast_mode(
+            "WARNING: `--session` and `--no-build` are set.\nUsing --session option value ({}) and ignoring `--no-build` option".format(session),  # noqa: E501
+        )
+
         is_no_build = False
 
     try:
@@ -355,11 +368,12 @@ def parse(report: str) -> Generator[CaseEventType, None, None]:
                 try:
                     xml = JUnitXml.fromfile(report, f)
                 except Exception as e:
-                    click.echo(click.style("Warning: error reading JUnitXml file {filename}: {error}".format(
-                        filename=report, error=e), fg="yellow"), err=True)
                     # `JUnitXml.fromfile()` will raise `JUnitXmlError` and other lxml related errors
                     # if the file has wrong format.
                     # https://github.com/weiwei/junitparser/blob/master/junitparser/junitparser.py#L321
+                    warn_and_exit_if_fail_fast_mode(
+                        "Warning: error reading JUnitXml file {filename}: {error}".format(
+                            filename=report, error=e))
                     return
                 if isinstance(xml, JUnitXml):
                     testsuites = [suite for suite in xml]
@@ -373,8 +387,9 @@ def parse(report: str) -> Generator[CaseEventType, None, None]:
                         for case in suite:
                             yield CaseEvent.from_case_and_suite(self.path_builder, case, suite, report, self.metadata_builder)
                 except Exception as e:
-                    click.echo(click.style("Warning: error parsing JUnitXml file {filename}: {error}".format(
-                        filename=report, error=e), fg="yellow"), err=True)
+                    warn_and_exit_if_fail_fast_mode(
+                        "Warning: error parsing JUnitXml file {filename}: {error}".format(
+                            filename=report, error=e))
 
             self.parse_func = parse
 
@@ -504,25 +519,12 @@ def send(payload: Dict[str, Union[str, List]]) -> None:
                 if res.status_code == HTTPStatus.NOT_FOUND:
                     if session:
                         build, _ = parse_session(session)
-                        click.echo(
-                            click.style(
-                                "Session {} was not found. "
-                                "Make sure to run `launchable record session --build {}` "
-                                "before `launchable record tests`".format(
-                                    session,
-                                    build),
-                                'yellow'),
-                            err=True)
+                        warn_and_exit_if_fail_fast_mode(
+                            "Session {} was not found. Make sure to run `launchable record session --build {}` before `launchable record tests`".format(session, build))  # noqa: E501
+
                     elif build_name:
-                        click.echo(
-                            click.style(
-                                "Build {} was not found. "
-                                "Make sure to run `launchable record build --name {}` "
-                                "before `launchable record tests`".format(
-                                    build_name,
-                                    build_name),
-                                'yellow'),
-                            err=True)
+                        warn_and_exit_if_fail_fast_mode(
+                            "Build {} was not found. Make sure to run `launchable record build --name {}` before `launchable record tests`".format(build_name, build_name))  # noqa: E501
 
                 res.raise_for_status()
 
@@ -606,18 +608,15 @@ def recorded_result() -> Tuple[int, int, int, float]:
 
             if count == 0:
                 if len(self.skipped_reports) != 0:
-                    click.echo(click.style(
+                    warn_and_exit_if_fail_fast_mode(
                         "{} test report(s) were skipped because they were created before this build was recorded.\n"
                         "Make sure to run your tests after you run `launchable record build`.\n"
-                        "Otherwise, if these are really correct test reports, use the `--allow-test-before-build` option.".format(
-                            len(self.skipped_reports)), 'yellow'))
+                        "Otherwise, if these are really correct test reports, use the `--allow-test-before-build` option.".
+                        format(len(self.skipped_reports)))
                     return
                 else:
-                    click.echo(
-                        click.style(
-                            "Looks like tests didn't run? "
-                            "If not, make sure the right files/directories were passed into `launchable record tests`",
-                            'yellow'))
+                    warn_and_exit_if_fail_fast_mode(
+                        "Looks like tests didn't run? If not, make sure the right files/directories were passed into `launchable record tests`")  # noqa: E501
                     return
 
             file_count = len(self.reports)

launchable/commands/subset.py

@@ -17,7 +17,10 @@
 from ..app import Application
 from ..testpath import FilePathNormalizer, TestPath
 from ..utils.click import DURATION, KEY_VALUE, PERCENTAGE, DurationType, PercentageType, ignorable_error
+from ..utils.commands import Command
 from ..utils.env_keys import REPORT_ERROR_KEY
+from ..utils.fail_fast_mode import (FailFastModeValidateParams, fail_fast_mode_validate,
+                                    set_fail_fast_mode, warn_and_exit_if_fail_fast_mode)
 from ..utils.launchable_client import LaunchableClient
 from .helper import find_or_create_session
 from .test_path_writer import TestPathWriter
@@ -225,7 +228,23 @@ def subset(
     test_suite: Optional[str] = None,
 ):
     app = context.obj
-    tracking_client = TrackingClient(Tracking.Command.SUBSET, app=app)
+    tracking_client = TrackingClient(Command.SUBSET, app=app)
+    client = LaunchableClient(
+        test_runner=context.invoked_subcommand,
+        app=app,
+        tracking_client=tracking_client)
+
+    set_fail_fast_mode(client.is_fail_fast_mode())
+    fail_fast_mode_validate(FailFastModeValidateParams(
+        command=Command.SUBSET,
+        session=session,
+        build=build_name,
+        flavor=flavor,
+        is_observation=is_observation,
+        links=links,
+        is_no_build=is_no_build,
+        test_suite=test_suite,
+    ))
 
     if is_observation and is_output_exclusion_rules:
         msg = (
@@ -262,15 +281,12 @@ def subset(
             sys.exit(1)
 
     if is_no_build and session:
-        click.echo(
-            click.style(
-                "WARNING: `--session` and `--no-build` are set.\nUsing --session option value ({}) and ignoring `--no-build` option".format(session),  # noqa: E501
-                fg='yellow'),
-            err=True)
+        warn_and_exit_if_fail_fast_mode(
+            "WARNING: `--session` and `--no-build` are set.\nUsing --session option value ({}) and ignoring `--no-build` option".format(session))  # noqa: E501
         is_no_build = False
 
     session_id = None
-    tracking_client = TrackingClient(Tracking.Command.SUBSET, app=app)
+
     try:
         if session_name:
             if not build_name:
@@ -410,12 +426,10 @@ def stdin(self) -> Union[TextIO, List]:
             they didn't feed anything from stdin
             """
             if sys.stdin.isatty():
-                click.echo(
-                    click.style(
-                        "Warning: this command reads from stdin but it doesn't appear to be connected to anything. "
-                        "Did you forget to pipe from another command?",
-                        fg='yellow'),
-                    err=True)
+                warn_and_exit_if_fail_fast_mode(
+                    "Warning: this command reads from stdin but it doesn't appear to be connected to anything. "
+                    "Did you forget to pipe from another command?"
+                )
             return sys.stdin
 
         @staticmethod
@@ -537,10 +551,6 @@ def run(self):
             else:
                 try:
                     test_runner = context.invoked_subcommand
-                    client = LaunchableClient(
-                        test_runner=test_runner,
-                        app=app,
-                        tracking_client=tracking_client)
 
                     # temporarily extend the timeout because subset API response has become slow
                     # TODO: remove this line when API response return respose
@@ -589,7 +599,7 @@ def run(self):
                         e, "Warning: the service failed to subset. Falling back to running all tests")
 
             if len(original_subset) == 0:
-                click.echo(click.style("Error: no tests found matching the path.", 'yellow'), err=True)
+                warn_and_exit_if_fail_fast_mode("Error: no tests found matching the path.")
                 return
 
             if split:

launchable/commands/verify.py

@@ -12,6 +12,7 @@
 
 from ..utils.authentication import get_org_workspace
 from ..utils.click import emoji
+from ..utils.commands import Command
 from ..utils.java import get_java_command
 from ..utils.launchable_client import LaunchableClient
 from ..version import __version__ as version
@@ -61,7 +62,7 @@ def verify(context: click.core.Context):
     # Click gracefully.
 
     org, workspace = get_org_workspace()
-    tracking_client = TrackingClient(Tracking.Command.VERIFY, app=context.obj)
+    tracking_client = TrackingClient(Command.VERIFY, app=context.obj)
     client = LaunchableClient(tracking_client=tracking_client, app=context.obj)
     java = get_java_command()
 

launchable/utils/commands.py

@@ -0,0 +1,13 @@
+from enum import Enum
+
+
+class Command(Enum):
+    VERIFY = 'VERIFY'
+    RECORD_TESTS = 'RECORD_TESTS'
+    RECORD_BUILD = 'RECORD_BUILD'
+    RECORD_SESSION = 'RECORD_SESSION'
+    SUBSET = 'SUBSET'
+    COMMIT = 'COMMIT'
+
+    def display_name(self):
+        return self.value.lower().replace('_', ' ')