Adding bootstrap --test-mode

--test-mode enables resilient bootstrap processing that continues building packages even when individual builds fail, instead of stopping at the first error.

When a package fails to build from source, it attempts to download a pre-built wheel as a fallback, and if both fail, records the failure and continues processing remaining packages.

At the end, it generates JSON reports (test-mode-failures.json and test-mode-summary.json) containing all failure details for automation and CI/CD integration.

Closes #713

Signed-off-by: Lalatendu Mohanty <lmohanty@redhat.com>

Author: LalatenduMohanty

src/fromager/bootstrapper.py

@@ -56,6 +56,47 @@ class SourceBuildResult:
     source_type: SourceType
 
 
+@dataclasses.dataclass
+class BuildFailure:
+    """Tracks a failed build in test mode for reporting.
+
+    Contains only fields needed for failure tracking and JSON serialization.
+    """
+
+    req: Requirement
+    resolved_version: Version | None = None
+    source_url_type: str = "unknown"
+    exception_type: str | None = None
+    exception_message: str | None = None
+
+    @classmethod
+    def from_exception(
+        cls,
+        req: Requirement,
+        resolved_version: Version | None,
+        source_url_type: str,
+        exception: Exception,
+    ) -> BuildFailure:
+        """Create a BuildFailure from an exception."""
+        return cls(
+            req=req,
+            resolved_version=resolved_version,
+            source_url_type=source_url_type,
+            exception_type=exception.__class__.__name__,
+            exception_message=str(exception),
+        )
+
+    def to_dict(self) -> dict[str, typing.Any]:
+        """Convert to JSON-serializable dict."""
+        return {
+            "package": str(self.req),
+            "version": str(self.resolved_version) if self.resolved_version else None,
+            "source_url_type": self.source_url_type,
+            "exception_type": self.exception_type,
+            "exception_message": self.exception_message,
+        }
+
+
 class Bootstrapper:
     def __init__(
         self,
@@ -64,12 +105,19 @@ def __init__(
         prev_graph: DependencyGraph | None = None,
         cache_wheel_server_url: str | None = None,
         sdist_only: bool = False,
+        test_mode: bool = False,
     ) -> None:
+        if test_mode and sdist_only:
+            raise ValueError(
+                "--test-mode requires full wheel builds; incompatible with --sdist-only"
+            )
+
         self.ctx = ctx
         self.progressbar = progressbar or progress.Progressbar(None)
         self.prev_graph = prev_graph
         self.cache_wheel_server_url = cache_wheel_server_url or ctx.wheel_server_url
         self.sdist_only = sdist_only
+        self.test_mode = test_mode
         self.why: list[tuple[RequirementType, Requirement, Version]] = []
         # Push items onto the stack as we start to resolve their
         # dependencies so at the end we have a list of items that need to
@@ -89,6 +137,9 @@ def __init__(
 
         self._build_order_filename = self.ctx.work_dir / "build-order.json"
 
+        # Track failed builds in test mode
+        self.failed_builds: list[BuildFailure] = []
+
     def resolve_version(
         self,
         req: Requirement,
@@ -145,6 +196,56 @@ def _processing_build_requirement(self, current_req_type: RequirementType) -> bo
         return False
 
     def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version:
+        """Bootstrap a package and its dependencies.
+
+        In test mode, catches all exceptions, records failures, and continues.
+        In normal mode, raises exceptions immediately (fail-fast).
+        """
+        try:
+            return self._bootstrap_impl(req, req_type)
+        except Exception as err:
+            if not self.test_mode:
+                raise
+            return self._handle_bootstrap_failure(req, err)
+
+    def _handle_bootstrap_failure(self, req: Requirement, err: Exception) -> Version:
+        """Handle complete bootstrap failure in test mode.
+
+        Called when an exception escapes from _bootstrap_impl(). Records the
+        failure and returns a version to allow processing to continue.
+        """
+        logger.error(
+            "test mode: failed to bootstrap %s: %s",
+            req,
+            err,
+            exc_info=True,
+        )
+
+        # Try to get resolved version from cache if available
+        cached = self._resolved_requirements.get(str(req))
+        resolved_version = cached[1] if cached else None
+
+        # Get source type if we have a resolved version
+        if resolved_version:
+            source_url_type = str(sources.get_source_type(self.ctx, req))
+        else:
+            source_url_type = "unknown"
+
+        self.failed_builds.append(
+            BuildFailure.from_exception(
+                req=req,
+                resolved_version=resolved_version,
+                source_url_type=source_url_type,
+                exception=err,
+            )
+        )
+
+        # Return a version to allow processing to continue
+        # Use cached version if available, otherwise return a null version
+        return resolved_version or Version("0.0.0")
+
+    def _bootstrap_impl(self, req: Requirement, req_type: RequirementType) -> Version:
+        """Internal implementation of bootstrap logic."""
         logger.info(f"bootstrapping {req} as {req_type} dependency of {self.why[-1:]}")
         constraint = self.ctx.constraints.get_constraint(req.name)
         if constraint:
@@ -217,15 +318,31 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version:
             )
 
             # Build from source (download, prepare, build wheel/sdist)
-            build_result = self._build_from_source(
-                req=req,
-                resolved_version=resolved_version,
-                source_url=source_url,
-                build_sdist_only=build_sdist_only,
-                cached_wheel_filename=cached_wheel_filename,
-                unpacked_cached_wheel=unpacked_cached_wheel,
-            )
+            try:
+                build_result = self._build_from_source(
+                    req=req,
+                    resolved_version=resolved_version,
+                    source_url=source_url,
+                    build_sdist_only=build_sdist_only,
+                    cached_wheel_filename=cached_wheel_filename,
+                    unpacked_cached_wheel=unpacked_cached_wheel,
+                )
+
+            except Exception as build_error:
+                if not self.test_mode:
+                    raise
+
+                fallback_result = self._handle_test_mode_failure(
+                    req=req,
+                    resolved_version=resolved_version,
+                    req_type=req_type,
+                    build_error=build_error,
+                )
+                if fallback_result is None:
+                    self.why.pop()
+                    return resolved_version
 
+                build_result, resolved_version = fallback_result
         hooks.run_post_bootstrap_hooks(
             ctx=self.ctx,
             req=req,
@@ -605,6 +722,90 @@ def _build_from_source(
             source_type=source_type,
         )
 
+    def _handle_test_mode_failure(
+        self,
+        req: Requirement,
+        resolved_version: Version,
+        req_type: RequirementType,
+        build_error: Exception,
+    ) -> tuple[SourceBuildResult, Version] | None:
+        """Handle build failure in test mode.
+
+        Attempts pre-built wheel fallback. Returns SourceBuildResult and
+        fallback version if successful, None if both build and fallback failed.
+
+        Args:
+            req: Package requirement
+            resolved_version: Version that failed to build
+            req_type: Type of requirement
+            build_error: The original build exception
+
+        Returns:
+            Tuple of (SourceBuildResult, fallback_version) if fallback succeeded,
+            None otherwise.
+        """
+        logger.warning(
+            "test mode: build failed for %s==%s, attempting pre-built fallback",
+            req.name,
+            resolved_version,
+            exc_info=True,
+        )
+
+        try:
+            wheel_url, fallback_version = self._resolve_prebuilt_with_history(
+                req=req,
+                req_type=req_type,
+            )
+
+            if fallback_version != resolved_version:
+                logger.warning(
+                    "test mode: version mismatch for %s - requested %s, fallback %s",
+                    req.name,
+                    resolved_version,
+                    fallback_version,
+                )
+
+            wheel_filename, unpack_dir = self._download_prebuilt(
+                req=req,
+                req_type=req_type,
+                resolved_version=fallback_version,
+                wheel_url=wheel_url,
+            )
+
+            logger.info(
+                "test mode: successfully used pre-built wheel for %s==%s",
+                req.name,
+                fallback_version,
+            )
+
+            build_result = SourceBuildResult(
+                wheel_filename=wheel_filename,
+                sdist_filename=None,
+                unpack_dir=unpack_dir,
+                sdist_root_dir=None,
+                build_env=None,
+                source_type=SourceType.PREBUILT,
+            )
+            return (build_result, fallback_version)
+
+        except Exception as fallback_error:
+            logger.error(
+                "test mode: pre-built fallback also failed for %s: %s",
+                req.name,
+                fallback_error,
+                exc_info=True,
+            )
+            source_url_type = str(sources.get_source_type(self.ctx, req))
+            self.failed_builds.append(
+                BuildFailure.from_exception(
+                    req=req,
+                    resolved_version=resolved_version,
+                    source_url_type=source_url_type,
+                    exception=build_error,
+                )
+            )
+            return None
+
     def _look_for_existing_wheel(
         self,
         req: Requirement,
@@ -1127,3 +1328,43 @@ def _add_to_build_order(
             # Requirement and Version instances that can't be
             # converted to JSON without help.
             json.dump(self._build_stack, f, indent=2, default=str)
+
+    def write_test_mode_report(self, work_dir: pathlib.Path) -> None:
+        """Write test mode failure report to JSON files.
+
+        Generates two JSON files:
+        - test-mode-failures.json: Detailed list of all failures
+        - test-mode-summary.json: Summary statistics
+        """
+        if not self.test_mode:
+            return
+
+        failures_file = work_dir / "test-mode-failures.json"
+        summary_file = work_dir / "test-mode-summary.json"
+
+        # Generate failures report
+        failures_data = {
+            "failures": [build_result.to_dict() for build_result in self.failed_builds]
+        }
+
+        with open(failures_file, "w") as f:
+            json.dump(failures_data, f, indent=2)
+        logger.info("test mode: wrote failure details to %s", failures_file)
+
+        # Generate summary report
+        exception_counts: dict[str, int] = {}
+        for build_result in self.failed_builds:
+            exception_type = build_result.exception_type or "Unknown"
+            exception_counts[exception_type] = (
+                exception_counts.get(exception_type, 0) + 1
+            )
+
+        summary_data = {
+            "total_packages": len(self._build_stack),
+            "total_failures": len(self.failed_builds),
+            "failure_breakdown": exception_counts,
+        }
+
+        with open(summary_file, "w") as f:
+            json.dump(summary_data, f, indent=2)
+        logger.info("test mode: wrote summary to %s", summary_file)

src/fromager/commands/bootstrap.py

@@ -97,6 +97,13 @@ def _get_requirements_from_args(
     default=False,
     help="Skip generating constraints.txt file to allow building collections with conflicting versions",
 )
+@click.option(
+    "--test-mode",
+    "test_mode",
+    is_flag=True,
+    default=False,
+    help="Test mode: mark failed packages as pre-built and continue, report failures at end",
+)
 @click.argument("toplevel", nargs=-1)
 @click.pass_obj
 def bootstrap(
@@ -106,6 +113,7 @@ def bootstrap(
     cache_wheel_server_url: str | None,
     sdist_only: bool,
     skip_constraints: bool,
+    test_mode: bool,
     toplevel: list[str],
 ) -> None:
     """Compute and build the dependencies of a set of requirements recursively
@@ -135,6 +143,11 @@ def bootstrap(
     else:
         logger.info("build all missing wheels")
 
+    if test_mode:
+        logger.info(
+            "test mode enabled: will mark failed packages as pre-built and continue"
+        )
+
     pre_built = wkctx.settings.list_pre_built()
     if pre_built:
         logger.info("treating %s as pre-built wheels", sorted(pre_built))
@@ -148,6 +161,7 @@ def bootstrap(
             prev_graph,
             cache_wheel_server_url,
             sdist_only=sdist_only,
+            test_mode=test_mode,
         )
 
         # we need to resolve all the top level dependencies before we start bootstrapping.
@@ -183,9 +197,28 @@ def bootstrap(
 
         for req in to_build:
             token = requirement_ctxvar.set(req)
-            bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL)
-            progressbar.update()
-            requirement_ctxvar.reset(token)
+            try:
+                bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL)
+                progressbar.update()
+            finally:
+                requirement_ctxvar.reset(token)
+
+        if test_mode:
+            bt.write_test_mode_report(wkctx.work_dir)
+            if bt.failed_builds:
+                logger.error(
+                    "test mode: %d package(s) failed to build",
+                    len(bt.failed_builds),
+                )
+                for build_result in bt.failed_builds:
+                    logger.error(
+                        "  - %s: %s",
+                        build_result.req,
+                        build_result.exception_type,
+                    )
+                raise SystemExit(1)
+            else:
+                logger.info("test mode: all packages processed successfully")
 
     constraints_filename = wkctx.work_dir / "constraints.txt"
     if skip_constraints:
@@ -480,6 +513,9 @@ def bootstrap_parallel(
     remaining wheels in parallel. The bootstrap step downloads sdists
     and builds build-time dependency in serial. The build-parallel step
     builds the remaining wheels in parallel.
+
+    Note: --test-mode is not supported in parallel builds. Use the serial
+    bootstrap command for test mode.
     """
     # Do not remove build environments in bootstrap phase to speed up the
     # parallel build phase.