feat: Add --test-mode for resilient bootstrap with failure handling

src/fromager/commands/bootstrap.py

@@ -97,6 +97,13 @@ def _get_requirements_from_args(
     default=False,
     help="Skip generating constraints.txt file to allow building collections with conflicting versions",
 )
+@click.option(
+    "--test-mode",
+    "test_mode",
+    is_flag=True,
+    default=False,
+    help="Test mode: mark failed packages as pre-built and continue, report failures at end",
+)
 @click.argument("toplevel", nargs=-1)
 @click.pass_obj
 def bootstrap(
@@ -106,6 +113,7 @@ def bootstrap(
     cache_wheel_server_url: str | None,
     sdist_only: bool,
     skip_constraints: bool,
+    test_mode: bool,
     toplevel: list[str],
 ) -> None:
     """Compute and build the dependencies of a set of requirements recursively
@@ -116,6 +124,11 @@ def bootstrap(
     """
     logger.info(f"cache wheel server url: {cache_wheel_server_url}")
 
+    if test_mode:
+        logger.info(
+            "test mode enabled: will mark failed packages as pre-built and continue"
+        )
+
     to_build = _get_requirements_from_args(toplevel, requirements_files)
     if not to_build:
         raise RuntimeError(
@@ -148,6 +161,7 @@ def bootstrap(
             prev_graph,
             cache_wheel_server_url,
             sdist_only=sdist_only,
+            test_mode=test_mode,
         )
 
         # we need to resolve all the top level dependencies before we start bootstrapping.
@@ -183,9 +197,29 @@ def bootstrap(
 
         for req in to_build:
             token = requirement_ctxvar.set(req)
-            bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL)
-            progressbar.update()
-            requirement_ctxvar.reset(token)
+            try:
+                bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL)
+                progressbar.update()
+                if test_mode:
+                    logger.info("Successfully processed: %s", req)
+            except Exception as err:
+                if test_mode:
+                    # Test mode: record error, log, and continue processing
+                    logger.error(
+                        "test mode: failed to process %s: %s",
+                        req,
+                        err,
+                        exc_info=True,  # Full traceback to debug log
+                    )
+                    bt.failed_builds.append(
+                        bootstrapper.BuildResult.failure(req=req, exception=err)
+                    )
+                    progressbar.update()  # Update progress even on failure
+                else:
+                    # Normal mode: re-raise the exception (fail-fast)
+                    raise
+            finally:
+                requirement_ctxvar.reset(token)
 
     constraints_filename = wkctx.work_dir / "constraints.txt"
     if skip_constraints:
@@ -200,7 +234,57 @@ def bootstrap(
 
     logger.debug("match_py_req LRU cache: %r", resolver.match_py_req.cache_info())
 
-    metrics.summarize(wkctx, "Bootstrapping")
+    # Test mode summary reporting
+    if test_mode:
+        if bt.failed_builds:
+            # Use repository's logging pattern for error reporting
+            logger.error("test mode: the following packages failed to build:")
+            for failure in sorted(
+                bt.failed_builds, key=lambda f: str(f.req) if f.req else ""
+            ):
+                if failure.req and failure.resolved_version:
+                    logger.error(
+                        "  - %s==%s",
+                        failure.req,
+                        failure.resolved_version,
+                    )
+                    if failure.exception_type:
+                        logger.error(
+                            "    Error: %s: %s",
+                            failure.exception_type,
+                            failure.exception_message,
+                        )
+                else:
+                    logger.error("  - unknown package (missing context)")
+
+            # Categorize failures by exception type for better analysis
+            failure_types: dict[str, list[str]] = {}
+            for failure in bt.failed_builds:
+                exc_type = failure.exception_type or "Unknown"
+                pkg_name = (
+                    f"{failure.req}=={failure.resolved_version}"
+                    if failure.req and failure.resolved_version
+                    else "unknown"
+                )
+                failure_types.setdefault(exc_type, []).append(pkg_name)
+
+            logger.error("")
+            logger.error("test mode: failure breakdown by type:")
+            for exc_type, packages in sorted(failure_types.items()):
+                logger.error("  %s: %d package(s)", exc_type, len(packages))
+
+            logger.error(
+                "test mode: %d package(s) failed to build", len(bt.failed_builds)
+            )
+            # Follow repository's error exit pattern like __main__.py and lint.py
+            raise SystemExit(
+                f"Test mode completed with {len(bt.failed_builds)} build failures"
+            )
+        else:
+            logger.info("test mode: all packages built successfully")
+        metrics.summarize(wkctx, "Test Mode Bootstrapping")
+    else:
+        metrics.summarize(wkctx, "Bootstrapping")
 
 
 def write_constraints_file(
@@ -480,6 +564,9 @@ def bootstrap_parallel(
     remaining wheels in parallel. The bootstrap step downloads sdists
     and builds build-time dependency in serial. The build-parallel step
     builds the remaining wheels in parallel.
+
+    Note: --test-mode is not supported with bootstrap-parallel. Use
+    'bootstrap --test-mode' for comprehensive failure testing.
     """
     # Do not remove build environments in bootstrap phase to speed up the
     # parallel build phase.
@@ -495,6 +582,7 @@ def bootstrap_parallel(
         cache_wheel_server_url=cache_wheel_server_url,
         sdist_only=True,
         skip_constraints=skip_constraints,
+        test_mode=False,
         toplevel=toplevel,
     )
 

src/fromager/commands/graph.py

@@ -125,7 +125,7 @@ def _find_customized_nodes(
     """Filter nodes to find only those with customizations."""
     customized_nodes: list[DependencyNode] = []
     for node in nodes:
-        pbi = wkctx.settings.package_build_info(node.canonicalized_name)
+        pbi = wkctx.package_build_info(node.canonicalized_name)
         if node.canonicalized_name != ROOT and pbi.has_customizations:
             customized_nodes.append(node)
     return customized_nodes
@@ -161,7 +161,7 @@ def _find_customized_dependencies_for_node(
                 continue
 
             child = edge.destination_node
-            child_pbi = wkctx.settings.package_build_info(child.canonicalized_name)
+            child_pbi = wkctx.package_build_info(child.canonicalized_name)
             new_path = path + [current_node.key]
 
             # Use the first requirement we encounter in the path
@@ -277,7 +277,7 @@ def get_node_id(node: str) -> str:
         if not name:
             node_type.append("toplevel")
         else:
-            pbi = wkctx.settings.package_build_info(name)
+            pbi = wkctx.package_build_info(name)
             all_patches: PatchMap = pbi.get_all_patches()
 
             if node.pre_built:

src/fromager/commands/list_overrides.py

@@ -65,7 +65,7 @@ def list_overrides(
     export_data = []
 
     for name in overridden_packages:
-        pbi = wkctx.settings.package_build_info(name)
+        pbi = wkctx.package_build_info(name)
         ps = wkctx.settings.package_setting(name)
 
         plugin_hooks: list[str] = []

src/fromager/context.py

@@ -164,7 +164,7 @@ def package_build_info(
             name = package.name
         else:
             name = package
-        return self.settings.package_build_info(name)
+        return self.settings.package_build_info(name, self)
 
     def setup(self) -> None:
         # The work dir must already exist, so don't try to create it.

src/fromager/hooks.py

@@ -3,6 +3,7 @@
 import logging
 import pathlib
 import typing
+from importlib import metadata
 
 from packaging.requirements import Requirement
 from stevedore import extension, hook
@@ -39,7 +40,7 @@ def _get_hooks(name: str) -> hook.HookManager:
 def log_hooks() -> None:
     # We load the hooks differently here because we want all of them when
     # normally we would load them by name.
-    _mgr = extension.ExtensionManager(
+    _mgr: extension.ExtensionManager[typing.Any] = extension.ExtensionManager(
         namespace="fromager.hooks",
         invoke_on_load=False,
         on_load_failure_callback=_die_on_plugin_load_failure,
@@ -56,9 +57,9 @@ def log_hooks() -> None:
 
 
 def _die_on_plugin_load_failure(
-    mgr: hook.HookManager,
-    ep: extension.Extension,
-    err: Exception,
+    mgr: hook.HookManager[typing.Any],
+    ep: metadata.EntryPoint,
+    err: BaseException,
 ) -> typing.NoReturn:
     raise RuntimeError(f"failed to load overrides for {ep.name}") from err
 

src/fromager/overrides.py

@@ -18,7 +18,7 @@
 _mgr = None
 
 
-def _get_extensions() -> extension.ExtensionManager:
+def _get_extensions() -> extension.ExtensionManager[typing.Any]:
     global _mgr
     if _mgr is None:
         _mgr = extension.ExtensionManager(
@@ -30,9 +30,9 @@ def _get_extensions() -> extension.ExtensionManager:
 
 
 def _die_on_plugin_load_failure(
-    mgr: extension.ExtensionManager,
-    ep: extension.Extension,
-    err: Exception,
+    mgr: extension.ExtensionManager[typing.Any],
+    ep: metadata.EntryPoint,
+    err: BaseException,
 ) -> None:
     raise RuntimeError(f"failed to load overrides for {ep.name}") from err
 

src/fromager/packagesettings.py

@@ -624,12 +624,26 @@ def get_available_memory_gib() -> float:
 
 
 class PackageBuildInfo:
-    """Package build information
+    """Variant-aware package build configuration and metadata.
 
-    Public API for PackageSettings with i
+    Primary public API for accessing package-specific settings during the build
+    process. Combines static configuration from YAML files with runtime context
+    to provide variant-specific (cpu, cuda, etc.) build information.
+
+    Key responsibilities:
+    - Determine if package should be built or use pre-built wheels
+    - Provide patches to apply for specific versions
+    - Configure build environment (parallel jobs, environment variables)
+    - Manage package customizations (plugins, custom download URLs)
+    - Calculate build tags from changelogs for wheel versioning
+
+    Instances are cached per package and accessed via ``WorkContext.package_build_info()``.
     """
 
-    def __init__(self, settings: Settings, ps: PackageSettings) -> None:
+    def __init__(
+        self, settings: Settings, ps: PackageSettings, ctx: context.WorkContext
+    ) -> None:
+        self._ctx = ctx
         self._variant = typing.cast(Variant, settings.variant)
         self._patches_dir = settings.patches_dir
         self._variant_changelog = settings.variant_changelog()
@@ -744,7 +758,7 @@ def has_customizations(self) -> bool:
 
     @property
     def pre_built(self) -> bool:
-        """Does the variant use pre-build wheels?"""
+        """Does the variant use pre-built wheels?"""
         vi = self._ps.variants.get(self.variant)
         if vi is not None:
             return vi.pre_built
@@ -1146,23 +1160,27 @@ def package_setting(self, package: str | Package) -> PackageSettings:
             self._package_settings[package] = ps
         return ps
 
-    def package_build_info(self, package: str | Package) -> PackageBuildInfo:
+    def package_build_info(
+        self, package: str | Package, ctx: context.WorkContext
+    ) -> PackageBuildInfo:
         """Get (cached) PackageBuildInfo for package and current variant"""
         package = Package(canonicalize_name(package, validate=True))
         pbi = self._pbi_cache.get(package)
         if pbi is None:
             ps = self.package_setting(package)
-            pbi = PackageBuildInfo(self, ps)
+            pbi = PackageBuildInfo(self, ps, ctx)
             self._pbi_cache[package] = pbi
         return pbi
 
     def list_pre_built(self) -> set[Package]:
-        """List packages marked as pre-built"""
-        return set(
-            name
-            for name in self._package_settings
-            if self.package_build_info(name).pre_built
-        )
+        """List packages marked as pre-built by configuration"""
+        result = set()
+        for name in self._package_settings:
+            ps = self._package_settings[name]
+            vi = ps.variants.get(self._variant)
+            if vi is not None and vi.pre_built:
+                result.add(name)
+        return result
 
     def list_overrides(self) -> set[Package]:
         """List packages with overrides