pre-upload.py

#!/usr/bin/env python3
# Copyright 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Repo pre-upload hook.

Normally this is loaded indirectly by repo itself, but it can be run directly
when developing.
"""

import argparse
import concurrent.futures
import datetime
import os
from pathlib import Path
import signal
import sys
from typing import List, Optional


# Assert some minimum Python versions as we don't test or support any others.
# See README.md for what version we may require.
if sys.version_info < (3, 6):
    print("repohooks: error: Python-3.6+ is required", file=sys.stderr)
    sys.exit(1)


THIS_FILE = Path(__file__).resolve()
THIS_DIR = THIS_FILE.parent
sys.path.insert(0, str(THIS_DIR.parent))

# We have to import our local modules after the sys.path tweak.  We can't use
# relative imports because this is an executable program, not a module.
# pylint: disable=wrong-import-position
import rh
import rh.config
import rh.git
import rh.hooks
import rh.results
import rh.terminal
import rh.utils


# Repohooks homepage.
REPOHOOKS_URL = "https://android.googlesource.com/platform/tools/repohooks/"


class Output(object):
    """Class for reporting hook status."""

    COLOR = rh.terminal.Color()
    COMMIT = COLOR.color(COLOR.CYAN, "COMMIT")
    RUNNING = COLOR.color(COLOR.YELLOW, "RUNNING")
    PASSED = COLOR.color(COLOR.GREEN, "PASSED")
    FAILED = COLOR.color(COLOR.RED, "FAILED")
    WARNING = COLOR.color(COLOR.YELLOW, "WARNING")
    FIXUP = COLOR.color(COLOR.MAGENTA, "FIXUP")

    # How long a hook is allowed to run before we warn that it is "too slow".
    _SLOW_HOOK_DURATION = datetime.timedelta(seconds=30)

    def __init__(self, project_name):
        """Create a new Output object for a specified project.

        Args:
            project_name: name of project.
        """
        self.project_name = project_name
        self.hooks = None
        self.num_hooks = None
        self.num_commits = None
        self.commit_index = 0
        self.success = True
        self.start_time = datetime.datetime.now()
        self.hook_start_time = None
        # Cache number of invisible characters in our banner.
        self._banner_esc_chars = len(self.COLOR.color(self.COLOR.YELLOW, ""))

    def set_num_commits(self, num_commits: int) -> None:
        """Keep track of how many commits we'll be running.

        Args:
            num_commits: Number of commits to be run.
        """
        self.num_commits = num_commits
        self.commit_index = 1

    def commit_start(self, hooks, commit, commit_summary):
        """Emit status for new commit.

        Args:
            hooks: All the hooks to be run for this commit.
            commit: commit hash.
            commit_summary: commit summary.
        """
        status_line = (
            f"[{self.COMMIT} "
            f"{self.commit_index}/{self.num_commits} "
            f"{commit[0:12]}] {commit_summary}"
        )
        rh.terminal.print_status_line(status_line, print_newline=True)
        self.commit_index += 1

        # Initialize the pending hooks line too.
        self.hooks = set(hooks)
        self.num_hooks = len(hooks)
        self.hook_banner()

    def hook_banner(self):
        """Display the banner for current set of hooks."""
        pending = ", ".join(x.name for x in self.hooks)
        status_line = (
            f"[{self.RUNNING} "
            f"{self.num_hooks - len(self.hooks)}/{self.num_hooks}] "
            f"{pending}"
        )
        if self._banner_esc_chars and sys.stderr.isatty():
            cols = os.get_terminal_size(sys.stderr.fileno()).columns
            status_line = status_line[0 : cols + self._banner_esc_chars]
        rh.terminal.print_status_line(status_line)

    def hook_finish(self, hook, duration):
        """Finish processing any per-hook state."""
        self.hooks.remove(hook)
        if duration >= self._SLOW_HOOK_DURATION:
            d = rh.utils.timedelta_str(duration)
            self.hook_warning(
                hook,
                f"This hook took {d} to finish which is fairly slow for "
                "developers.\nPlease consider moving the check to the "
                "server/CI system instead.",
            )

        # Show any hooks still pending.
        if self.hooks:
            self.hook_banner()

    def hook_error(self, hook, error):
        """Print an error for a single hook.

        Args:
            hook: The hook that generated the output.
            error: error string.
        """
        self.error(f"{hook.name} hook", error)

    def hook_warning(self, hook, warning):
        """Print a warning for a single hook.

        Args:
            hook: The hook that generated the output.
            warning: warning string.
        """
        status_line = f"[{self.WARNING}] {hook.name}"
        rh.terminal.print_status_line(status_line, print_newline=True)
        print(warning, file=sys.stderr)

    def error(self, header, error):
        """Print a general error.

        Args:
            header: A unique identifier for the source of this error.
            error: error string.
        """
        status_line = f"[{self.FAILED}] {header}"
        rh.terminal.print_status_line(status_line, print_newline=True)
        print(error, file=sys.stderr)
        self.success = False

    def hook_fixups(
        self,
        project_results: rh.results.ProjectResults,
        hook_results: List[rh.results.HookResult],
    ) -> None:
        """Display summary of possible fixups for a single hook."""
        for result in (x for x in hook_results if x.fixup_cmd):
            cmd = result.fixup_cmd + list(result.files)
            for line in (
                f"[{self.FIXUP}] {result.hook} has automated fixups available",
                f"  cd {rh.shell.quote(project_results.workdir)} && \\",
                f"    {rh.shell.cmd_to_str(cmd)}",
            ):
                rh.terminal.print_status_line(line, print_newline=True)

    def finish(self):
        """Print summary for all the hooks."""
        header = self.PASSED if self.success else self.FAILED
        status = "passed" if self.success else "failed"
        d = rh.utils.timedelta_str(datetime.datetime.now() - self.start_time)
        rh.terminal.print_status_line(
            f"[{header}] repohooks for {self.project_name} {status} in {d}",
            print_newline=True,
        )


def _process_hook_results(results):
    """Returns an error string if an error occurred.

    Args:
        results: A list of HookResult objects, or None.

    Returns:
        error output if an error occurred, otherwise None
        warning output if an error occurred, otherwise None
    """
    if not results:
        return (None, None)

    # We track these as dedicated fields in case a hook doesn't output anything.
    # We want to treat silent non-zero exits as failures too.
    has_error = False
    has_warning = False

    error_ret = ""
    warning_ret = ""
    for result in results:
        if result or result.is_warning():
            ret = ""
            if result.files:
                ret += f"  FILES: {rh.shell.cmd_to_str(result.files)}\n"
            lines = result.error.splitlines()
            ret += "\n".join(f"    {x}" for x in lines)
            if result.is_warning():
                has_warning = True
                warning_ret += ret
            else:
                has_error = True
                error_ret += ret

    return (
        error_ret if has_error else None,
        warning_ret if has_warning else None,
    )


def _get_project_config(from_git=False):
    """Returns the configuration for a project.

    Expects to be called from within the project root.

    Args:
        from_git: If true, we are called from git directly and repo should not
            be used.
    """
    if from_git:
        global_paths = (rh.git.find_repo_root(),)
    else:
        global_paths = (
            # Load the global config found in the manifest repo.
            (os.path.join(rh.git.find_repo_root(), ".repo", "manifests")),
            # Load the global config found in the root of the repo checkout.
            rh.git.find_repo_root(),
        )

    paths = (
        # Load the config for this git repo.
        ".",
    )
    return rh.config.PreUploadSettings(paths=paths, global_paths=global_paths)


def _attempt_fixes(projects_results: List[rh.results.ProjectResults]) -> None:
    """Attempts to fix fixable results."""
    # Filter out any result that has a fixup.
    fixups = []
    for project_results in projects_results:
        fixups.extend(
            (project_results.workdir, x) for x in project_results.fixups
        )
    if not fixups:
        return

    if len(fixups) > 1:
        banner = f"Multiple fixups ({len(fixups)}) are available."
    else:
        banner = "Automated fixups are available."
    print(Output.COLOR.color(Output.COLOR.MAGENTA, banner), file=sys.stderr)

    # If there's more than one fixup available, ask if they want to blindly run
    # them all, or prompt for them one-by-one.
    mode = "some"
    if len(fixups) > 1:
        while True:
            response = rh.terminal.str_prompt(
                "What would you like to do",
                ("Run (A)ll", "Run (S)ome", "(D)ry-run", "(N)othing [default]"),
            )
            if not response:
                print("", file=sys.stderr)
                return
            if response.startswith("a") or response.startswith("y"):
                mode = "all"
                break
            elif response.startswith("s"):
                mode = "some"
                break
            elif response.startswith("d"):
                mode = "dry-run"
                break
            elif response.startswith("n"):
                print("", file=sys.stderr)
                return

    # Walk all the fixups and run them one-by-one.
    for workdir, result in fixups:
        if mode == "some":
            if not rh.terminal.boolean_prompt(
                f"Run {result.hook} fixup for {result.commit}"
            ):
                continue

        cmd = tuple(result.fixup_cmd) + tuple(result.files)
        print(
            f"\n[{Output.RUNNING}] cd {rh.shell.quote(workdir)} && "
            f"{rh.shell.cmd_to_str(cmd)}",
            file=sys.stderr,
        )
        if mode == "dry-run":
            continue

        cmd_result = rh.utils.run(cmd, cwd=workdir, check=False)
        if cmd_result.returncode:
            print(
                f"[{Output.WARNING}] command exited {cmd_result.returncode}",
                file=sys.stderr,
            )
        else:
            print(f"[{Output.PASSED}] great success", file=sys.stderr)

    print(
        f"\n[{Output.FIXUP}] Please amend & rebase your tree before "
        "attempting to upload again.\n",
        file=sys.stderr,
    )


def _run_project_hooks_in_cwd(
    project_name: str,
    proj_dir: str,
    output: Output,
    jobs: Optional[int] = None,
    from_git: bool = False,
    commit_list: Optional[List[str]] = None,
) -> rh.results.ProjectResults:
    """Run the project-specific hooks in the cwd.

    Args:
        project_name: The name of this project.
        proj_dir: The directory for this project (for passing on in metadata).
        output: Helper for summarizing output/errors to the user.
        jobs: How many hooks to run in parallel.
        from_git: If true, we are called from git directly and repo should not
            be used.
        commit_list: A list of commits to run hooks against.  If None or empty
            list then we'll automatically get the list of commits that would be
            uploaded.

    Returns:
        All the results for this project.
    """
    ret = rh.results.ProjectResults(project_name, proj_dir, [])

    try:
        config = _get_project_config(from_git)
    except rh.config.ValidationError as e:
        output.error("Loading config files", str(e))
        return ret._replace(internal_failure=True)

    builtin_hooks = list(config.callable_builtin_hooks())
    custom_hooks = list(config.callable_custom_hooks())

    # If the repo has no pre-upload hooks enabled, then just return.
    if not builtin_hooks and not custom_hooks:
        return ret

    # Set up the environment like repo would with the forall command.
    try:
        remote = rh.git.get_upstream_remote()
        upstream_branch = rh.git.get_upstream_branch()
    except rh.utils.CalledProcessError as e:
        output.error(
            "Upstream remote/tracking branch lookup",
            f"{e}\nDid you run repo start?  Is your HEAD detached?",
        )
        return ret._replace(internal_failure=True)

    project = rh.Project(name=project_name, dir=proj_dir)
    rel_proj_dir = os.path.relpath(proj_dir, rh.git.find_repo_root())

    # Filter out the hooks to process.
    builtin_hooks = [x for x in builtin_hooks if rel_proj_dir not in x.scope]
    custom_hooks = [x for x in custom_hooks if rel_proj_dir not in x.scope]

    if not builtin_hooks and not custom_hooks:
        return ret

    os.environ.update(
        {
            "REPO_LREV": rh.git.get_commit_for_ref(upstream_branch),
            "REPO_PATH": rel_proj_dir,
            "REPO_PROJECT": project_name,
            "REPO_REMOTE": remote,
            "REPO_RREV": rh.git.get_remote_revision(upstream_branch, remote),
        }
    )

    if not commit_list:
        commit_list = rh.git.get_commits(
            ignore_merged_commits=config.ignore_merged_commits
        )
    output.set_num_commits(len(commit_list))

    def _run_hook(hook, project, commit, desc, diff):
        """Run a hook, gather stats, and process its results."""
        start = datetime.datetime.now()
        results = hook.hook(project, commit, desc, diff)
        (error, warning) = _process_hook_results(results)
        duration = datetime.datetime.now() - start
        return (hook, results, error, warning, duration)

    with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as executor:
        for commit in commit_list:
            # Mix in some settings for our hooks.
            os.environ["PREUPLOAD_COMMIT"] = commit
            diff = rh.git.get_affected_files(commit)
            desc = rh.git.get_commit_desc(commit)
            os.environ["PREUPLOAD_COMMIT_MESSAGE"] = desc

            commit_summary = desc.split("\n", 1)[0]
            output.commit_start(
                builtin_hooks + custom_hooks, commit, commit_summary
            )

            def run_hooks(hooks):
                # pylint: disable=cell-var-from-loop
                futures = (
                    executor.submit(
                        _run_hook, hook, project, commit, desc, diff
                    )
                    for hook in hooks
                )
                future_results = (
                    x.result() for x in concurrent.futures.as_completed(futures)
                )
                for (
                    hook,
                    hook_results,
                    error,
                    warning,
                    duration,
                ) in future_results:
                    ret.add_results(hook_results)
                    if error is not None or warning is not None:
                        if warning is not None:
                            output.hook_warning(hook, warning)
                        if error is not None:
                            output.hook_error(hook, error)
                            output.hook_fixups(ret, hook_results)
                    output.hook_finish(hook, duration)

            run_hooks(builtin_hooks)
            run_hooks(custom_hooks)

    return ret


def _run_project_hooks(
    project_name: str,
    proj_dir: Optional[str] = None,
    jobs: Optional[int] = None,
    from_git: bool = False,
    commit_list: Optional[List[str]] = None,
) -> rh.results.ProjectResults:
    """Run the project-specific hooks in |proj_dir|.

    Args:
        project_name: The name of project to run hooks for.
        proj_dir: If non-None, this is the directory the project is in.  If
            None, we'll ask repo.
        jobs: How many hooks to run in parallel.
        from_git: If true, we are called from git directly and repo should not
            be used.
        commit_list: A list of commits to run hooks against.  If None or empty
            list then we'll automatically get the list of commits that would be
            uploaded.

    Returns:
        All the results for this project.
    """
    output = Output(project_name)

    if proj_dir is None:
        cmd = ["repo", "forall", project_name, "-c", "pwd"]
        result = rh.utils.run(cmd, capture_output=True)
        proj_dirs = result.stdout.split()
        if not proj_dirs:
            print(f"{project_name} cannot be found.", file=sys.stderr)
            print("Please specify a valid project.", file=sys.stderr)
            return False
        if len(proj_dirs) > 1:
            print(
                f"{project_name} is associated with multiple directories.",
                file=sys.stderr,
            )
            print(
                "Please specify a directory to help disambiguate.",
                file=sys.stderr,
            )
            return False
        proj_dir = proj_dirs[0]

    pwd = os.getcwd()
    try:
        # Hooks assume they are run from the root of the project.
        os.chdir(proj_dir)
        return _run_project_hooks_in_cwd(
            project_name,
            proj_dir,
            output,
            jobs=jobs,
            from_git=from_git,
            commit_list=commit_list,
        )
    finally:
        output.finish()
        os.chdir(pwd)


def _run_projects_hooks(
    project_list: List[str],
    worktree_list: List[Optional[str]],
    jobs: Optional[int] = None,
    from_git: bool = False,
    commit_list: Optional[List[str]] = None,
) -> bool:
    """Run all the hooks

    Args:
        project_list: List of project names.
        worktree_list: List of project checkouts.
        jobs: How many hooks to run in parallel.
        from_git: If true, we are called from git directly and repo should not
            be used.
        commit_list: A list of commits to run hooks against.  If None or empty
            list then we'll automatically get the list of commits that would be
            uploaded.

    Returns:
        True if everything passed, else False.
    """
    results = []
    for project, worktree in zip(project_list, worktree_list):
        result = _run_project_hooks(
            project,
            proj_dir=worktree,
            jobs=jobs,
            from_git=from_git,
            commit_list=commit_list,
        )
        results.append(result)
        if result:
            # If a repo had failures, add a blank line to help break up the
            # output.  If there were no failures, then the output should be
            # very minimal, so we don't add it then.
            print("", file=sys.stderr)

    _attempt_fixes(results)
    return not any(results)


def main(project_list, worktree_list=None, **_kwargs):
    """Main function invoked directly by repo.

    We must use the name "main" as that is what repo requires.

    This function will exit directly upon error so that repo doesn't print some
    obscure error message.

    Args:
        project_list: List of projects to run on.
        worktree_list: A list of directories.  It should be the same length as
            project_list, so that each entry in project_list matches with a
            directory in worktree_list.  If None, we will attempt to calculate
            the directories automatically.
        kwargs: Leave this here for forward-compatibility.
    """
    if not worktree_list:
        worktree_list = [None] * len(project_list)
    if not _run_projects_hooks(project_list, worktree_list):
        color = rh.terminal.Color()
        print(
            color.color(color.RED, "FATAL")
            + ": Preupload failed due to above error(s).\n"
            f"For more info, see: {REPOHOOKS_URL}",
            file=sys.stderr,
        )
        sys.exit(1)


def _identify_project(path, from_git=False):
    """Identify the repo project associated with the given path.

    Returns:
        A string indicating what project is associated with the path passed in
        or a blank string upon failure.
    """
    if from_git:
        cmd = ["git", "rev-parse", "--show-toplevel"]
        project_path = rh.utils.run(cmd, capture_output=True).stdout.strip()
        cmd = ["git", "rev-parse", "--show-superproject-working-tree"]
        superproject_path = rh.utils.run(
            cmd, capture_output=True
        ).stdout.strip()
        module_path = project_path[len(superproject_path) + 1 :]
        cmd = [
            "git",
            "config",
            "-f",
            ".gitmodules",
            "--name-only",
            "--get-regexp",
            r"^submodule\..*\.path$",
            f"^{module_path}$",
        ]
        module_name = rh.utils.run(
            cmd, cwd=superproject_path, capture_output=True
        ).stdout.strip()
        return module_name[len("submodule.") : -len(".path")]
    else:
        cmd = ["repo", "forall", ".", "-c", "echo ${REPO_PROJECT}"]
        return rh.utils.run(cmd, capture_output=True, cwd=path).stdout.strip()


def direct_main(argv):
    """Run hooks directly (outside of the context of repo).

    Args:
        argv: The command line args to process.

    Returns:
        0 if no pre-upload failures, 1 if failures.

    Raises:
        BadInvocation: On some types of invocation errors.
    """
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--git",
        action="store_true",
        help="This hook is called from git instead of repo",
    )
    parser.add_argument(
        "--dir",
        default=None,
        help="The directory that the project lives in.  If not "
        "specified, use the git project root based on the cwd.",
    )
    parser.add_argument(
        "--project",
        default=None,
        help="The project repo path; this can affect how the "
        "hooks get run, since some hooks are project-specific."
        "If not specified, `repo` will be used to figure this "
        "out based on the dir.",
    )
    parser.add_argument(
        "-j",
        "--jobs",
        type=int,
        help="Run up to this many hooks in parallel. Setting "
        "to 1 forces serial execution, and the default "
        "automatically chooses an appropriate number for the "
        "current system.",
    )
    parser.add_argument("commits", nargs="*", help="Check specific commits")
    opts = parser.parse_args(argv)

    # Check/normalize git dir; if unspecified, we'll use the root of the git
    # project from CWD.
    if opts.dir is None:
        cmd = ["git", "rev-parse", "--git-dir"]
        git_dir = rh.utils.run(cmd, capture_output=True).stdout.strip()
        if not git_dir:
            parser.error("The current directory is not part of a git project.")
        opts.dir = os.path.dirname(os.path.abspath(git_dir))
    elif not os.path.isdir(opts.dir):
        parser.error(f"Invalid dir: {opts.dir}")
    elif not rh.git.is_git_repository(opts.dir):
        parser.error(f"Not a git repository: {opts.dir}")

    # Identify the project if it wasn't specified; this _requires_ the repo
    # tool to be installed and for the project to be part of a repo checkout.
    if not opts.project:
        opts.project = _identify_project(opts.dir, opts.git)
        if not opts.project:
            parser.error(f"Couldn't identify the project of {opts.dir}")

    try:
        if _run_projects_hooks(
            [opts.project],
            [opts.dir],
            jobs=opts.jobs,
            from_git=opts.git,
            commit_list=opts.commits,
        ):
            return 0
    except KeyboardInterrupt:
        print("Aborting execution early due to user interrupt", file=sys.stderr)
        return 128 + signal.SIGINT
    return 1


if __name__ == "__main__":
    sys.exit(direct_main(sys.argv[1:]))