diff --git a/WORKSPACE b/WORKSPACE index c87249e92c0..a7c7b306bb0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,5 +1 @@ workspace(name = "workerd") - -load("//build/deps:dep_pyodide.bzl", "dep_pyodide") - -dep_pyodide() diff --git a/build/deps/dep_pyodide.bzl b/build/deps/dep_pyodide.bzl index 7e9e40057d3..a70afb0c0cc 100644 --- a/build/deps/dep_pyodide.bzl +++ b/build/deps/dep_pyodide.bzl @@ -53,38 +53,6 @@ def _capnp_bundle(id = None, integrity = None, **_kwds): url = PYODIDE_CAPN_BIN + "pyodide_%s.capnp.bin" % id, ) -def _snapshot_http_file(bundle_name, folder, snapshot, integrity, hash, r2_base = PYODIDE_CAPN_BIN): - if not snapshot: - return - if not integrity: - fail("Snapshot %s from bundle %s has missing integrity" % (snapshot, bundle_name)) - if folder == "baseline-snapshot/": - key = hash - else: - key = snapshot - http_file( - name = "pyodide-snapshot-" + snapshot, - integrity = integrity, - url = r2_base + folder + key, - ) - -def _snapshot_http_files( - name, - baseline_snapshot = None, - baseline_snapshot_hash = None, - baseline_snapshot_integrity = None, - numpy_snapshot = None, - numpy_snapshot_integrity = None, - fastapi_snapshot = None, - fastapi_snapshot_integrity = None, - dedicated_fastapi_snapshot = None, - dedicated_fastapi_snapshot_integrity = None, - **_kwds): - _snapshot_http_file(name, "baseline-snapshot/", baseline_snapshot, baseline_snapshot_integrity, baseline_snapshot_hash) - _snapshot_http_file(name, "test-snapshot/", numpy_snapshot, numpy_snapshot_integrity, None) - _snapshot_http_file(name, "test-snapshot/", fastapi_snapshot, fastapi_snapshot_integrity, None) - _snapshot_http_file(name, "", dedicated_fastapi_snapshot, dedicated_fastapi_snapshot_integrity, None, VENDOR_R2) - def dep_pyodide(): for info in PYODIDE_VERSIONS: _pyodide_core(**info) @@ -97,5 +65,50 @@ def dep_pyodide(): for info in PYTHON_LOCKFILES: _pyodide_packages(**info) + # Accumulator to de-duplicate generated http_file rules. + gen_http_file = {} + + def _snapshot_http_files( + name, + baseline_snapshot = None, + baseline_snapshot_hash = None, + baseline_snapshot_integrity = None, + numpy_snapshot = None, + numpy_snapshot_integrity = None, + fastapi_snapshot = None, + fastapi_snapshot_integrity = None, + dedicated_fastapi_snapshot = None, + dedicated_fastapi_snapshot_integrity = None, + **_kwds): + def _snapshot_http_file(bundle_name, folder, snapshot, integrity, hash, r2_base = PYODIDE_CAPN_BIN): + if not snapshot: + return + if not integrity: + fail("Snapshot %s from bundle %s has missing integrity" % (snapshot, bundle_name)) + if folder == "baseline-snapshot/": + key = hash + else: + key = snapshot + gen_http_file[snapshot] = struct( + integrity = integrity, + url = r2_base + folder + key, + ) + + _snapshot_http_file(name, "baseline-snapshot/", baseline_snapshot, baseline_snapshot_integrity, baseline_snapshot_hash) + _snapshot_http_file(name, "test-snapshot/", numpy_snapshot, numpy_snapshot_integrity, None) + _snapshot_http_file(name, "test-snapshot/", fastapi_snapshot, fastapi_snapshot_integrity, None) + _snapshot_http_file(name, "", dedicated_fastapi_snapshot, dedicated_fastapi_snapshot_integrity, None, VENDOR_R2) + for ver in BUNDLE_VERSION_INFO.values(): _snapshot_http_files(**ver) + for snapshot, info in gen_http_file.items(): + http_file( + name = "pyodide-snapshot-" + snapshot, + integrity = info.integrity, + url = info.url, + ) + +def _impl(module_ctx): + dep_pyodide() + +pyodide = module_extension(implementation = _impl) diff --git a/build/deps/python.MODULE.bazel b/build/deps/python.MODULE.bazel index 6eb29b25837..2e7540f54d7 100644 --- a/build/deps/python.MODULE.bazel +++ b/build/deps/python.MODULE.bazel @@ -18,3 +18,57 @@ pip.parse( requirements_lock = "//build/deps:requirements.txt", ) use_repo(pip, "py_deps", "v8_python_deps") + +pyodide = use_extension("//build/deps:dep_pyodide.bzl", "pyodide") + +[ + use_repo( + pyodide, + repo.format(version = version), + ) + for repo in [ + "beautifulsoup4_src_{version}", + "fastapi_src_{version}", + "pyodide-{version}", + "python-workers-runtime-sdk_src_{version}", + "shapely_src_{version}", + ] + for version in [ + "0.26.0a2", + "0.28.2", + "development", + ] + # Exclusions: some packages are absent in some versions + if repo.format(version = version) not in [ + "pyodide-development", + "shapely_src_0.26.0a2", + ] +] + +[ + use_repo( + pyodide, + repo.format(version = version), + ) + for repo in [ + "all_pyodide_wheels_{version}", + "pyodide-lock_{version}.json", + ] + for version in [ + "20240829.4", + "20250808", + ] +] + +use_repo( + pyodide, + "pyodide-snapshot-baseline-4569679fb.bin", + "pyodide-snapshot-baseline-61eedf943.bin", + "pyodide-snapshot-ew-py-package-snapshot_fastapi-v2.bin", + "pyodide-snapshot-ew-py-package-snapshot_numpy-v2.bin", + "pyodide-snapshot-package_snapshot_fastapi-a6ccb56fe.bin", + "pyodide-snapshot-package_snapshot_numpy-60c9cb28e.bin", + "pyodide-snapshot-snapshot_a6b652a95810783f5078b9a5dbd4a07c30718acb4ff724e82c25db7353dd7f2d.bin", + "pyodide_0.26.0a2_2024-03-01_75.capnp.bin", + "pyodide_0.28.2_2025-01-16_6.capnp.bin", +) diff --git a/build/is_pyodide_bzlmod.bzl b/build/is_pyodide_bzlmod.bzl new file mode 100644 index 00000000000..c9ae4142429 --- /dev/null +++ b/build/is_pyodide_bzlmod.bzl @@ -0,0 +1,2 @@ +# workerd uses bzlmod for pyodide (and everything else) +is_pyodide_bzlmod = True diff --git a/src/workerd/server/tests/python/py_wd_test.bzl b/src/workerd/server/tests/python/py_wd_test.bzl index 23f1a7fb2f6..780853d1651 100644 --- a/src/workerd/server/tests/python/py_wd_test.bzl +++ b/src/workerd/server/tests/python/py_wd_test.bzl @@ -1,3 +1,4 @@ +load("@//:build/is_pyodide_bzlmod.bzl", "is_pyodide_bzlmod") load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@bazel_skylib//rules:expand_template.bzl", "expand_template") load("//:build/python_metadata.bzl", "BUNDLE_VERSION_INFO") @@ -32,7 +33,15 @@ def _py_wd_test_helper( pkg_tag = BUNDLE_VERSION_INFO[python_flag]["packages"] data = data + ["@all_pyodide_wheels_%s//:whls" % pkg_tag] - args = args + ["--pyodide-package-disk-cache-dir", "../all_pyodide_wheels_%s" % pkg_tag] + args = args + ["--pyodide-package-disk-cache-dir"] + + # TODO(cleanup): We support both mangled and non-mangled wheels paths for now, clean up once + # downstream repo fully uses bzlmod + if is_pyodide_bzlmod: + # +pyodide+ is a bzlmod canonical repository name + args.append("../+pyodide+all_pyodide_wheels_%s" % pkg_tag) + else: + args.append("../all_pyodide_wheels_%s" % pkg_tag) load_snapshot = None pyodide_version = BUNDLE_VERSION_INFO[python_flag]["real_pyodide_version"]