fix: better bootstrap manifest handling

BUILD.bazel

@@ -43,6 +43,7 @@ filegroup(
         "internal_dev_deps.bzl",
         "internal_dev_setup.bzl",
         "version.bzl",
+        "//command_line_option:distribution",
         "//python:distribution",
         "//tools:distribution",
     ],

CHANGELOG.md

@@ -74,6 +74,8 @@ END_UNRELEASED_TEMPLATE
   in order to make `pytorch` and friends easier to patch.
 * (wheel) `py_wheel` no longer expands the input depset during analysis,
   improving analysis performance for targets with large dependency trees.
+* (binaries/tests) (Windows) `--enable_runfiles=true` is the default for
+  py_binary/py_test.
 
 {#v0-0-0-fixed}
 ### Fixed

command_line_option/BUILD.bazel

@@ -0,0 +1,25 @@
+# Special placeholders for Bazel builtin //command_line_option psuedo-targets
+#
+# These are special targets to use with `py_binary.config_settings` that are
+# treated as aliases for `//command_line_option:XXX` psuedo-targets. They
+# are not actual flags or have any value.
+
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+alias(
+    name = "build_runfile_links",
+    actual = "//python:none",
+)
+
+alias(
+    name = "enable_runfiles",
+    actual = "//python:none",
+)
+
+filegroup(
+    name = "distribution",
+    srcs = glob(["**"]),
+    visibility = ["//:__subpackages__"],
+)

docs/api/rules_python/command_line_option/index.md

@@ -0,0 +1,41 @@
+:::{default-domain} bzl
+:::
+:::{bzl:currentfile} //command_line_option:BUILD.bazel
+:::
+
+# //command_line_option
+
+This package provides special targets that correspond to the Bazel-builtin
+`//command_line_option` psuedo-targets. These can be used with the {obj}`config_settings`
+attribute on Python rules to transition specific command line flags for a target.
+
+:::{note}
+These targets are not actual `alias()` targets, nor are they the actual builtin
+command line flags. They are regular targets that the `config_settings` transition
+logic specially recognizes and handles as if they were the builtin `//command_line_option`
+psuedo-targets.
+:::
+
+:::{seealso}
+The `config_settings` attribute documentation on:
+* {obj}`py_binary.config_settings`
+* {obj}`py_test.config_settings`
+:::
+
+## build_runfile_links
+
+:::{bzl:target} build_runfile_links
+
+Special target for the Bazel-builtin `//command_line_option:build_runfile_links` flag.
+
+See the [Bazel documentation for --build_runfile_links](https://bazel.build/reference/command-line-reference#flag--build_runfile_links).
+:::
+
+## enable_runfiles
+
+:::{bzl:target} enable_runfiles
+
+Special target for the Bazel-builtin `//command_line_option:enable_runfiles` flag.
+
+See the [Bazel documentation for --enable_runfiles](https://bazel.build/reference/command-line-reference#flag--enable_runfiles).
+:::

python/private/attributes.bzl

@@ -400,6 +400,23 @@ particular CPU, or defining a custom setting that `select()` uses elsewhere
 to pick between `pip.parse` hubs. See the [How to guide on multiple
 versions of a library] for a more concrete example.
 
+:::{important}
+Labels with package `command_line_option` are handled specially: they are treated
+as the Bazel-builtin `//command_line_option:<name>` psuedo-targets.
+
+e.g. `@foo//command_line_option:NAME` will attempt to transition
+the Bazel-builtin `//command_line_option:NAME` setting.
+
+See the {obj}`@rules_python//command_line_option` package for some predefined
+special targets, or define your own by putting them in your own `command_line_option`
+directory.
+:::
+
+:::{seealso}
+* {obj}`//command_line_option:build_runfile_links`
+* {obj}`//command_line_option:enable_runfiles`
+:::
+
 :::{note}
 These values are transitioned on, so will affect the analysis graph and the
 associated memory overhead. The more unique configurations in your overall
@@ -426,7 +443,11 @@ def apply_config_settings_attr(settings, attr):
         {type}`dict[str, object]` the input `settings` value.
     """
     for key, value in attr.config_settings.items():
-        settings[str(key)] = value
+        if key.package == "command_line_option":
+            str_key = "//command_line_option:" + key.name
+        else:
+            str_key = str(key)
+        settings[str_key] = value
     return settings
 
 AGNOSTIC_EXECUTABLE_ATTRS = dicts.add(

python/private/common_labels.bzl

@@ -8,15 +8,20 @@ labels = struct(
     ADD_SRCS_TO_RUNFILES = str(Label("//python/config_settings:add_srcs_to_runfiles")),
     BOOTSTRAP_IMPL = str(Label("//python/config_settings:bootstrap_impl")),
     BUILD_PYTHON_ZIP = str(Label("//python/config_settings:build_python_zip")),
+    # NOTE: Special target; see definition for details.
+    BUILD_RUNFILE_LINKS = str(Label("//command_line_option:build_runfile_links")),
     DEBUGGER = str(Label("//python/config_settings:debugger")),
+    # NOTE: Special target; see definition for details.
+    ENABLE_RUNFILES = str(Label("//command_line_option:enable_runfiles")),
     EXEC_TOOLS_TOOLCHAIN = str(Label("//python/config_settings:exec_tools_toolchain")),
-    PIP_ENV_MARKER_CONFIG = str(Label("//python/config_settings:pip_env_marker_config")),
     NONE = str(Label("//python:none")),
+    PIP_ENV_MARKER_CONFIG = str(Label("//python/config_settings:pip_env_marker_config")),
     PIP_WHL = str(Label("//python/config_settings:pip_whl")),
     PIP_WHL_GLIBC_VERSION = str(Label("//python/config_settings:pip_whl_glibc_version")),
     PIP_WHL_MUSLC_VERSION = str(Label("//python/config_settings:pip_whl_muslc_version")),
     PIP_WHL_OSX_ARCH = str(Label("//python/config_settings:pip_whl_osx_arch")),
     PIP_WHL_OSX_VERSION = str(Label("//python/config_settings:pip_whl_osx_version")),
+    PLATFORMS_OS_WINDOWS = str(Label("@platforms//os:windows")),
     PRECOMPILE = str(Label("//python/config_settings:precompile")),
     PRECOMPILE_SOURCE_RETENTION = str(Label("//python/config_settings:precompile_source_retention")),
     PYC_COLLECTION = str(Label("//python/config_settings:pyc_collection")),

python/private/py_binary_macro.bzl

@@ -14,11 +14,11 @@
 """Implementation of macro-half of py_binary rule."""
 
 load(":py_binary_rule.bzl", py_binary_rule = "py_binary")
-load(":py_executable.bzl", "convert_legacy_create_init_to_int")
+load(":py_executable.bzl", "common_executable_macro_kwargs_setup")
 
 def py_binary(**kwargs):
     py_binary_macro(py_binary_rule, **kwargs)
 
 def py_binary_macro(py_rule, **kwargs):
-    convert_legacy_create_init_to_int(kwargs)
+    common_executable_macro_kwargs_setup(kwargs)
     py_rule(**kwargs)

python/private/py_executable.bzl

@@ -1771,6 +1771,32 @@ def _create_run_environment_info(ctx, inherited_environment):
         inherited_environment = inherited_environment,
     )
 
+def add_config_setting_defaults(kwargs):
+    config_settings = kwargs.get("config_settings", None)
+    if config_settings == None:
+        config_settings = {}
+
+    # NOTE: This code runs in loading phase within the context of the caller.
+    # Label() must be used to resolve repo names within rules_python's
+    # context to avoid unknown repo name errors.
+    default = select({
+        labels.PLATFORMS_OS_WINDOWS: {
+            ##labels.BUILD_RUNFILE_LINKS: "true",
+            labels.ENABLE_RUNFILES: "true",
+        },
+        "//conditions:default": {
+            ##labels.BUILD_RUNFILE_LINKS: "true",
+        },
+    })
+
+    # Let user-provided settings have precedence
+    config_settings = default | config_settings
+    kwargs["config_settings"] = config_settings
+
+def common_executable_macro_kwargs_setup(kwargs):
+    convert_legacy_create_init_to_int(kwargs)
+    add_config_setting_defaults(kwargs)
+
 def _transition_executable_impl(settings, attr):
     settings = dict(settings)
     apply_config_settings_attr(settings, attr)
@@ -1780,6 +1806,7 @@ def _transition_executable_impl(settings, attr):
 
     if attr.stamp != -1:
         settings["//command_line_option:stamp"] = str(attr.stamp)
+
     return settings
 
 def create_executable_rule(*, attrs, **kwargs):
@@ -1833,10 +1860,14 @@ def create_executable_rule_builder(implementation, **kwargs):
             inputs = TRANSITION_LABELS + [
                 labels.PYTHON_VERSION,
                 "//command_line_option:stamp",
+                "//command_line_option:build_runfile_links",
+                "//command_line_option:enable_runfiles",
             ],
             outputs = TRANSITION_LABELS + [
                 labels.PYTHON_VERSION,
                 "//command_line_option:stamp",
+                "//command_line_option:build_runfile_links",
+                "//command_line_option:enable_runfiles",
             ],
         ),
         **kwargs

python/private/py_test_macro.bzl

@@ -13,12 +13,12 @@
 # limitations under the License.
 """Implementation of macro-half of py_test rule."""
 
-load(":py_executable.bzl", "convert_legacy_create_init_to_int")
+load(":py_executable.bzl", "common_executable_macro_kwargs_setup")
 load(":py_test_rule.bzl", py_test_rule = "py_test")
 
 def py_test(**kwargs):
     py_test_macro(py_test_rule, **kwargs)
 
 def py_test_macro(py_rule, **kwargs):
-    convert_legacy_create_init_to_int(kwargs)
+    common_executable_macro_kwargs_setup(kwargs)
     py_rule(**kwargs)

python/private/python_bootstrap_template.txt

@@ -10,13 +10,13 @@ import sys
 import os
 import subprocess
 import uuid
-# runfiles-relative path
 # NOTE: The sentinel strings are split (e.g., "%stage2" + "_bootstrap%") so that
 # the substitution logic won't replace them. This allows runtime detection of
 # unsubstituted placeholders, which occurs when native py_binary is used in
 # external repositories. In that case, we fall back to %main% which Bazel's
 # native rule does substitute.
 _STAGE2_BOOTSTRAP_SENTINEL = "%stage2" + "_bootstrap%"
+# runfiles-root-relative path
 STAGE2_BOOTSTRAP="%stage2_bootstrap%"
 
 # NOTE: The fallback logic from stage2_bootstrap to main is only present
@@ -165,6 +165,29 @@ def print_verbose(*args, mapping=None, values=None):
         else:
             print("bootstrap: stage 1:", *args, file=sys.stderr, flush=True)
 
+def maybe_find_in_manifest(rf_root_path, show_lines=False):
+    if not os.environ.get("RUNFILES_MANIFEST_FILE"):
+        return None
+    if not os.path.exists(os.environ["RUNFILES_MANIFEST_FILE"]):
+        return None
+    manifest_path = os.environ["RUNFILES_MANIFEST_FILE"]
+    print_verbose("search manifest for:",  rf_root_path)
+
+    # Add trailing space to avoid suffix-string matching
+    search_for_prefix = rf_root_path.encode("utf8") + b" "
+    # Use binary to avoid BOM issues on Windows
+    with open(manifest_path, 'rb') as fp:
+      for line in fp:
+        if show_lines:
+          print_verbose("manifest line:", repr(line))
+        # NOTE: This doesn't handle escaped manifest lines
+        if line.startswith(search_for_prefix):
+          _, _, main_filename = line.partition(b" ")
+          return main_filename.strip().decode("utf8")
+
+    return None
+
+
 def FindBinary(module_space, bin_name):
   """Finds the real binary if it's not a normal absolute path."""
   if not bin_name:
@@ -180,7 +203,13 @@ def FindBinary(module_space, bin_name):
   # Use normpath() to convert slashes to os.sep on Windows.
   elif os.sep in os.path.normpath(bin_name):
     # Case 3: Path is relative to the repo root.
-    return os.path.join(module_space, bin_name)
+    full_path = os.path.join(module_space, bin_name)
+    if os.path.exists(full_path):
+        return full_path
+    full_path = maybe_find_in_manifest(bin_name, True)
+    if not full_path:
+        raise AssertionError(f"Unable to find Python: {bin_name}")
+    return full_path
   else:
     # Case 4: Path has to be looked up in the search path.
     return SearchPath(bin_name)
@@ -233,6 +262,19 @@ def FindModuleSpace(main_rel_path):
 
   raise AssertionError('Cannot find .runfiles directory for %s' % sys.argv[0])
 
+def find_main_file(module_space, main_rel_path):
+  main_filename = os.path.join(module_space, main_rel_path)
+  main_filename = GetWindowsPathWithUNCPrefix(main_filename)
+
+  if os.path.exists(main_filename):
+      return main_filename
+
+  main_filename = maybe_find_in_manifest(STAGE2_BOOTSTRAP)
+  if main_filename:
+      return main_filename
+  raise AssertionError(f"Cannot find main filename: {main_rel_path}")
+
+
 def ExtractZip(zip_path, dest_dir):
   """Extracts the contents of a zip file, preserving the unix file mode bits.
 
@@ -410,8 +452,7 @@ def Main():
   # See: https://docs.python.org/3.11/using/cmdline.html#envvar-PYTHONSAFEPATH
   new_env['PYTHONSAFEPATH'] = '1'
 
-  main_filename = os.path.join(module_space, main_rel_path)
-  main_filename = GetWindowsPathWithUNCPrefix(main_filename)
+  main_filename = find_main_file(module_space, main_rel_path)
   assert os.path.exists(main_filename), \
          'Cannot exec() %r: file not found.' % main_filename
   assert os.access(main_filename, os.R_OK), \

python/private/stage2_bootstrap_template.py

@@ -194,6 +194,8 @@ def find_runfiles_root(main_rel_path):
         else:
             stub_filename = os.path.join(os.path.dirname(stub_filename), target)
 
+    # The `--enable_runfiles=false` flag is likely set, which isn't fully
+    # supported.
     raise AssertionError("Cannot find .runfiles directory for %s" % sys.argv[0])
 
 

tests/support/py_reconfig.bzl

@@ -47,16 +47,16 @@ def _perform_transition_impl(input_settings, attr, base_impl):
         settings[labels.VENVS_USE_DECLARE_SYMLINK] = attr.venvs_use_declare_symlink
     if attr.venvs_site_packages:
         settings[labels.VENVS_SITE_PACKAGES] = attr.venvs_site_packages
-    for key, value in attr.config_settings.items():
-        settings[str(key)] = value
     return settings
 
 _BUILTIN_BUILD_PYTHON_ZIP = [] if config.bazel_10_or_later else [
     "//command_line_option:build_python_zip",
 ]
 
 _RECONFIG_INPUTS = [
+    "//command_line_option:build_runfile_links",
     "//command_line_option:extra_toolchains",
+    "//command_line_option:stamp",
     CUSTOM_RUNTIME,
     labels.BOOTSTRAP_IMPL,
     labels.PYTHON_SRC,