diff --git a/BUILD.bazel b/BUILD.bazel index aa2642d43f..7da18ebaa4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -43,6 +43,7 @@ filegroup( "internal_dev_deps.bzl", "internal_dev_setup.bzl", "version.bzl", + "//command_line_option:distribution", "//python:distribution", "//tools:distribution", ], diff --git a/CHANGELOG.md b/CHANGELOG.md index 18234744bd..e1641cabe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,8 @@ END_UNRELEASED_TEMPLATE in order to make `pytorch` and friends easier to patch. * (wheel) `py_wheel` no longer expands the input depset during analysis, improving analysis performance for targets with large dependency trees. +* (binaries/tests) (Windows) `--enable_runfiles=true` is the default for + py_binary/py_test. {#v0-0-0-fixed} ### Fixed diff --git a/command_line_option/BUILD.bazel b/command_line_option/BUILD.bazel new file mode 100644 index 0000000000..a4d5f0f80d --- /dev/null +++ b/command_line_option/BUILD.bazel @@ -0,0 +1,25 @@ +# Special placeholders for Bazel builtin //command_line_option psuedo-targets +# +# These are special targets to use with `py_binary.config_settings` that are +# treated as aliases for `//command_line_option:XXX` psuedo-targets. They +# are not actual flags or have any value. + +package( + default_visibility = ["//visibility:public"], +) + +alias( + name = "build_runfile_links", + actual = "//python:none", +) + +alias( + name = "enable_runfiles", + actual = "//python:none", +) + +filegroup( + name = "distribution", + srcs = glob(["**"]), + visibility = ["//:__subpackages__"], +) diff --git a/docs/api/rules_python/command_line_option/index.md b/docs/api/rules_python/command_line_option/index.md new file mode 100644 index 0000000000..61059013e4 --- /dev/null +++ b/docs/api/rules_python/command_line_option/index.md @@ -0,0 +1,41 @@ +:::{default-domain} bzl +::: +:::{bzl:currentfile} //command_line_option:BUILD.bazel +::: + +# //command_line_option + +This package provides special targets that correspond to the Bazel-builtin +`//command_line_option` psuedo-targets. These can be used with the {obj}`config_settings` +attribute on Python rules to transition specific command line flags for a target. + +:::{note} +These targets are not actual `alias()` targets, nor are they the actual builtin +command line flags. They are regular targets that the `config_settings` transition +logic specially recognizes and handles as if they were the builtin `//command_line_option` +psuedo-targets. +::: + +:::{seealso} +The `config_settings` attribute documentation on: +* {obj}`py_binary.config_settings` +* {obj}`py_test.config_settings` +::: + +## build_runfile_links + +:::{bzl:target} build_runfile_links + +Special target for the Bazel-builtin `//command_line_option:build_runfile_links` flag. + +See the [Bazel documentation for --build_runfile_links](https://bazel.build/reference/command-line-reference#flag--build_runfile_links). +::: + +## enable_runfiles + +:::{bzl:target} enable_runfiles + +Special target for the Bazel-builtin `//command_line_option:enable_runfiles` flag. + +See the [Bazel documentation for --enable_runfiles](https://bazel.build/reference/command-line-reference#flag--enable_runfiles). +::: diff --git a/python/private/attributes.bzl b/python/private/attributes.bzl index 362eee8f2e..f368f2f40e 100644 --- a/python/private/attributes.bzl +++ b/python/private/attributes.bzl @@ -400,6 +400,23 @@ particular CPU, or defining a custom setting that `select()` uses elsewhere to pick between `pip.parse` hubs. See the [How to guide on multiple versions of a library] for a more concrete example. +:::{important} +Labels with package `command_line_option` are handled specially: they are treated +as the Bazel-builtin `//command_line_option:` psuedo-targets. + +e.g. `@foo//command_line_option:NAME` will attempt to transition +the Bazel-builtin `//command_line_option:NAME` setting. + +See the {obj}`@rules_python//command_line_option` package for some predefined +special targets, or define your own by putting them in your own `command_line_option` +directory. +::: + +:::{seealso} +* {obj}`//command_line_option:build_runfile_links` +* {obj}`//command_line_option:enable_runfiles` +::: + :::{note} These values are transitioned on, so will affect the analysis graph and the associated memory overhead. The more unique configurations in your overall @@ -426,7 +443,11 @@ def apply_config_settings_attr(settings, attr): {type}`dict[str, object]` the input `settings` value. """ for key, value in attr.config_settings.items(): - settings[str(key)] = value + if key.package == "command_line_option": + str_key = "//command_line_option:" + key.name + else: + str_key = str(key) + settings[str_key] = value return settings AGNOSTIC_EXECUTABLE_ATTRS = dicts.add( diff --git a/python/private/common_labels.bzl b/python/private/common_labels.bzl index 9c21198a62..b6594cf0b9 100644 --- a/python/private/common_labels.bzl +++ b/python/private/common_labels.bzl @@ -8,15 +8,20 @@ labels = struct( ADD_SRCS_TO_RUNFILES = str(Label("//python/config_settings:add_srcs_to_runfiles")), BOOTSTRAP_IMPL = str(Label("//python/config_settings:bootstrap_impl")), BUILD_PYTHON_ZIP = str(Label("//python/config_settings:build_python_zip")), + # NOTE: Special target; see definition for details. + BUILD_RUNFILE_LINKS = str(Label("//command_line_option:build_runfile_links")), DEBUGGER = str(Label("//python/config_settings:debugger")), + # NOTE: Special target; see definition for details. + ENABLE_RUNFILES = str(Label("//command_line_option:enable_runfiles")), EXEC_TOOLS_TOOLCHAIN = str(Label("//python/config_settings:exec_tools_toolchain")), - PIP_ENV_MARKER_CONFIG = str(Label("//python/config_settings:pip_env_marker_config")), NONE = str(Label("//python:none")), + PIP_ENV_MARKER_CONFIG = str(Label("//python/config_settings:pip_env_marker_config")), PIP_WHL = str(Label("//python/config_settings:pip_whl")), PIP_WHL_GLIBC_VERSION = str(Label("//python/config_settings:pip_whl_glibc_version")), PIP_WHL_MUSLC_VERSION = str(Label("//python/config_settings:pip_whl_muslc_version")), PIP_WHL_OSX_ARCH = str(Label("//python/config_settings:pip_whl_osx_arch")), PIP_WHL_OSX_VERSION = str(Label("//python/config_settings:pip_whl_osx_version")), + PLATFORMS_OS_WINDOWS = str(Label("@platforms//os:windows")), PRECOMPILE = str(Label("//python/config_settings:precompile")), PRECOMPILE_SOURCE_RETENTION = str(Label("//python/config_settings:precompile_source_retention")), PYC_COLLECTION = str(Label("//python/config_settings:pyc_collection")), diff --git a/python/private/py_binary_macro.bzl b/python/private/py_binary_macro.bzl index fa10f2e8a3..c7840eff55 100644 --- a/python/private/py_binary_macro.bzl +++ b/python/private/py_binary_macro.bzl @@ -14,11 +14,11 @@ """Implementation of macro-half of py_binary rule.""" load(":py_binary_rule.bzl", py_binary_rule = "py_binary") -load(":py_executable.bzl", "convert_legacy_create_init_to_int") +load(":py_executable.bzl", "common_executable_macro_kwargs_setup") def py_binary(**kwargs): py_binary_macro(py_binary_rule, **kwargs) def py_binary_macro(py_rule, **kwargs): - convert_legacy_create_init_to_int(kwargs) + common_executable_macro_kwargs_setup(kwargs) py_rule(**kwargs) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 7ff6278e02..041d6a21e1 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -1771,6 +1771,32 @@ def _create_run_environment_info(ctx, inherited_environment): inherited_environment = inherited_environment, ) +def add_config_setting_defaults(kwargs): + config_settings = kwargs.get("config_settings", None) + if config_settings == None: + config_settings = {} + + # NOTE: This code runs in loading phase within the context of the caller. + # Label() must be used to resolve repo names within rules_python's + # context to avoid unknown repo name errors. + default = select({ + labels.PLATFORMS_OS_WINDOWS: { + ##labels.BUILD_RUNFILE_LINKS: "true", + labels.ENABLE_RUNFILES: "true", + }, + "//conditions:default": { + ##labels.BUILD_RUNFILE_LINKS: "true", + }, + }) + + # Let user-provided settings have precedence + config_settings = default | config_settings + kwargs["config_settings"] = config_settings + +def common_executable_macro_kwargs_setup(kwargs): + convert_legacy_create_init_to_int(kwargs) + add_config_setting_defaults(kwargs) + def _transition_executable_impl(settings, attr): settings = dict(settings) apply_config_settings_attr(settings, attr) @@ -1780,6 +1806,7 @@ def _transition_executable_impl(settings, attr): if attr.stamp != -1: settings["//command_line_option:stamp"] = str(attr.stamp) + return settings def create_executable_rule(*, attrs, **kwargs): @@ -1833,10 +1860,14 @@ def create_executable_rule_builder(implementation, **kwargs): inputs = TRANSITION_LABELS + [ labels.PYTHON_VERSION, "//command_line_option:stamp", + "//command_line_option:build_runfile_links", + "//command_line_option:enable_runfiles", ], outputs = TRANSITION_LABELS + [ labels.PYTHON_VERSION, "//command_line_option:stamp", + "//command_line_option:build_runfile_links", + "//command_line_option:enable_runfiles", ], ), **kwargs diff --git a/python/private/py_test_macro.bzl b/python/private/py_test_macro.bzl index 028dee6678..bc58f859f8 100644 --- a/python/private/py_test_macro.bzl +++ b/python/private/py_test_macro.bzl @@ -13,12 +13,12 @@ # limitations under the License. """Implementation of macro-half of py_test rule.""" -load(":py_executable.bzl", "convert_legacy_create_init_to_int") +load(":py_executable.bzl", "common_executable_macro_kwargs_setup") load(":py_test_rule.bzl", py_test_rule = "py_test") def py_test(**kwargs): py_test_macro(py_test_rule, **kwargs) def py_test_macro(py_rule, **kwargs): - convert_legacy_create_init_to_int(kwargs) + common_executable_macro_kwargs_setup(kwargs) py_rule(**kwargs) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index f2d5a42fda..5aa4befde3 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -10,13 +10,13 @@ import sys import os import subprocess import uuid -# runfiles-relative path # NOTE: The sentinel strings are split (e.g., "%stage2" + "_bootstrap%") so that # the substitution logic won't replace them. This allows runtime detection of # unsubstituted placeholders, which occurs when native py_binary is used in # external repositories. In that case, we fall back to %main% which Bazel's # native rule does substitute. _STAGE2_BOOTSTRAP_SENTINEL = "%stage2" + "_bootstrap%" +# runfiles-root-relative path STAGE2_BOOTSTRAP="%stage2_bootstrap%" # NOTE: The fallback logic from stage2_bootstrap to main is only present @@ -165,6 +165,29 @@ def print_verbose(*args, mapping=None, values=None): else: print("bootstrap: stage 1:", *args, file=sys.stderr, flush=True) +def maybe_find_in_manifest(rf_root_path, show_lines=False): + if not os.environ.get("RUNFILES_MANIFEST_FILE"): + return None + if not os.path.exists(os.environ["RUNFILES_MANIFEST_FILE"]): + return None + manifest_path = os.environ["RUNFILES_MANIFEST_FILE"] + print_verbose("search manifest for:", rf_root_path) + + # Add trailing space to avoid suffix-string matching + search_for_prefix = rf_root_path.encode("utf8") + b" " + # Use binary to avoid BOM issues on Windows + with open(manifest_path, 'rb') as fp: + for line in fp: + if show_lines: + print_verbose("manifest line:", repr(line)) + # NOTE: This doesn't handle escaped manifest lines + if line.startswith(search_for_prefix): + _, _, main_filename = line.partition(b" ") + return main_filename.strip().decode("utf8") + + return None + + def FindBinary(module_space, bin_name): """Finds the real binary if it's not a normal absolute path.""" if not bin_name: @@ -180,7 +203,13 @@ def FindBinary(module_space, bin_name): # Use normpath() to convert slashes to os.sep on Windows. elif os.sep in os.path.normpath(bin_name): # Case 3: Path is relative to the repo root. - return os.path.join(module_space, bin_name) + full_path = os.path.join(module_space, bin_name) + if os.path.exists(full_path): + return full_path + full_path = maybe_find_in_manifest(bin_name, True) + if not full_path: + raise AssertionError(f"Unable to find Python: {bin_name}") + return full_path else: # Case 4: Path has to be looked up in the search path. return SearchPath(bin_name) @@ -233,6 +262,19 @@ def FindModuleSpace(main_rel_path): raise AssertionError('Cannot find .runfiles directory for %s' % sys.argv[0]) +def find_main_file(module_space, main_rel_path): + main_filename = os.path.join(module_space, main_rel_path) + main_filename = GetWindowsPathWithUNCPrefix(main_filename) + + if os.path.exists(main_filename): + return main_filename + + main_filename = maybe_find_in_manifest(STAGE2_BOOTSTRAP) + if main_filename: + return main_filename + raise AssertionError(f"Cannot find main filename: {main_rel_path}") + + def ExtractZip(zip_path, dest_dir): """Extracts the contents of a zip file, preserving the unix file mode bits. @@ -410,8 +452,7 @@ def Main(): # See: https://docs.python.org/3.11/using/cmdline.html#envvar-PYTHONSAFEPATH new_env['PYTHONSAFEPATH'] = '1' - main_filename = os.path.join(module_space, main_rel_path) - main_filename = GetWindowsPathWithUNCPrefix(main_filename) + main_filename = find_main_file(module_space, main_rel_path) assert os.path.exists(main_filename), \ 'Cannot exec() %r: file not found.' % main_filename assert os.access(main_filename, os.R_OK), \ diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index 959e631ad1..0e906669d9 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -194,6 +194,8 @@ def find_runfiles_root(main_rel_path): else: stub_filename = os.path.join(os.path.dirname(stub_filename), target) + # The `--enable_runfiles=false` flag is likely set, which isn't fully + # supported. raise AssertionError("Cannot find .runfiles directory for %s" % sys.argv[0]) diff --git a/tests/support/py_reconfig.bzl b/tests/support/py_reconfig.bzl index d0cb968466..9bbfdb1104 100644 --- a/tests/support/py_reconfig.bzl +++ b/tests/support/py_reconfig.bzl @@ -47,8 +47,6 @@ def _perform_transition_impl(input_settings, attr, base_impl): settings[labels.VENVS_USE_DECLARE_SYMLINK] = attr.venvs_use_declare_symlink if attr.venvs_site_packages: settings[labels.VENVS_SITE_PACKAGES] = attr.venvs_site_packages - for key, value in attr.config_settings.items(): - settings[str(key)] = value return settings _BUILTIN_BUILD_PYTHON_ZIP = [] if config.bazel_10_or_later else [ @@ -56,7 +54,9 @@ _BUILTIN_BUILD_PYTHON_ZIP = [] if config.bazel_10_or_later else [ ] _RECONFIG_INPUTS = [ + "//command_line_option:build_runfile_links", "//command_line_option:extra_toolchains", + "//command_line_option:stamp", CUSTOM_RUNTIME, labels.BOOTSTRAP_IMPL, labels.PYTHON_SRC,