From 50fcdb8939709bfe31cb2c794d5d865a16f30f07 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 11:35:07 +0100 Subject: [PATCH 01/27] Add macos-latest to versions matrix --- .github/workflows/versions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index db50e2a9..4e4f640b 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -53,7 +53,7 @@ jobs: max-parallel: 12 matrix: # TODO: fix for Mac and Windows. - os: [ubuntu-latest] + os: [ubuntu-latest, macos-latest] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: From ce22dc6e21b6c97dfb399afda8b6e3493c98bff7 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 13:31:47 +0100 Subject: [PATCH 02/27] Fix wheel architecture for MacOS and ignore .venv --- .github/workflows/versions.yml | 8 +------- .gitignore | 1 + 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index 4e4f640b..366052fd 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -149,13 +149,7 @@ jobs: poetry lock poetry install - - name: Build PyDP macOS - if: runner.os == 'macOS' - run: | - poetry run python setup.py build bdist_wheel --plat-name macosx_10_14_x86_64 - - - name: Build PyDP Linux / Windows - if: runner.os != 'macOS' + - name: Build PyDP run: | poetry run python setup.py build bdist_wheel diff --git a/.gitignore b/.gitignore index 0df50794..8c310c07 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ coverage_report/* !coverage_report/index.html .python-version +.venv/ From f315799e856ab66151013f6ec83b427d204a3c22 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 13:49:06 +0100 Subject: [PATCH 03/27] Install cmake in prereqs_mac.sh --- prereqs_mac.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/prereqs_mac.sh b/prereqs_mac.sh index c8da5112..b7309269 100755 --- a/prereqs_mac.sh +++ b/prereqs_mac.sh @@ -36,6 +36,14 @@ else brew install clang-format fi +# cmake +if command -v cmake &>/dev/null; then + echo "cmake already installed" +else + echo "installing cmake" + brew install cmake +fi + # poetry echo "Checking for poetry" if python3 -c "import poetry" &> /dev/null; then From dd4fccaff41249ffb9b8449dc4db26294010e058 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 14:00:07 +0100 Subject: [PATCH 04/27] Add missing Abseil logging libraries for macOS dynamic linkage --- src/bindings/PyDP/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index 7731f0d1..2bad8a0d 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -51,6 +51,10 @@ target_link_libraries(_pydp PUBLIC dp_algorithms dp_protos absl::statusor absl::strings absl::flat_hash_set + absl::raw_logging_internal + absl::log_internal_message + absl::log_internal_check_op + absl::log_globals ) if(NOT WIN32) From 83d9db8214abac8eff99120e927a7d53629ee026 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 14:27:09 +0100 Subject: [PATCH 05/27] Add remaining missing Abseil logging components (kSignedCharNull fix) to macOS linkage --- src/bindings/PyDP/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index 2bad8a0d..60587882 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -55,6 +55,9 @@ target_link_libraries(_pydp PUBLIC dp_algorithms dp_protos absl::log_internal_message absl::log_internal_check_op absl::log_globals + absl::log + absl::log_internal_nullguard + absl::log_internal_globals ) if(NOT WIN32) From 168d4388e24048fb69d1485e6385cee255cc2d28 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 14:48:11 +0100 Subject: [PATCH 06/27] Enforce native arm64 linking and macOS 14.0 deployment target for python wheels to prevent universal2 cross-compile linker breaks --- .github/workflows/publish.yml | 5 ++++- .github/workflows/versions.yml | 11 ++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8a6c6fb7..8b10ce59 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -122,7 +122,10 @@ jobs: - name: Build PyDP macOS wheel if: runner.os == 'macOS' run: | - poetry run python setup.py build bdist_wheel --plat-name macosx_10_14_x86_64 + poetry run python setup.py build bdist_wheel + env: + MACOSX_DEPLOYMENT_TARGET: "14.0" + ARCHFLAGS: "-arch arm64" - name: Build PyDP Windows wheel if: runner.os == 'Windows' diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index 366052fd..a37702c6 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -149,7 +149,16 @@ jobs: poetry lock poetry install - - name: Build PyDP + - name: Build PyDP (macOS) + if: runner.os == 'macOS' + run: | + poetry run python setup.py build bdist_wheel + env: + MACOSX_DEPLOYMENT_TARGET: "14.0" + ARCHFLAGS: "-arch arm64" + + - name: Build PyDP (Linux/Windows) + if: runner.os != 'macOS' run: | poetry run python setup.py build bdist_wheel From d71bfe0b8d3450bda473f2d84a97ed066f0fd7fb Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 14:56:54 +0100 Subject: [PATCH 07/27] Force deployment target 15.0 and explicitly disable universal2 cross-compilation wheel generation via _PYTHON_HOST_PLATFORM --- .github/workflows/publish.yml | 4 +++- .github/workflows/versions.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8b10ce59..12cb1d79 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -124,9 +124,11 @@ jobs: run: | poetry run python setup.py build bdist_wheel env: - MACOSX_DEPLOYMENT_TARGET: "14.0" + MACOSX_DEPLOYMENT_TARGET: "15.0" + _PYTHON_HOST_PLATFORM: "macosx-15.0-arm64" ARCHFLAGS: "-arch arm64" + - name: Build PyDP Windows wheel if: runner.os == 'Windows' run: | diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index a37702c6..e714a0d5 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -154,9 +154,11 @@ jobs: run: | poetry run python setup.py build bdist_wheel env: - MACOSX_DEPLOYMENT_TARGET: "14.0" + MACOSX_DEPLOYMENT_TARGET: "15.0" + _PYTHON_HOST_PLATFORM: "macosx-15.0-arm64" ARCHFLAGS: "-arch arm64" + - name: Build PyDP (Linux/Windows) if: runner.os != 'macOS' run: | From 0a0b8da920855facb93ee817f779f8068f9d682a Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 15:01:55 +0100 Subject: [PATCH 08/27] Temporarily narrow CI matrix to macos-latest and python 3.13 for faster debugging --- .github/workflows/versions.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index e714a0d5..639d02d5 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -53,8 +53,8 @@ jobs: max-parallel: 12 matrix: # TODO: fix for Mac and Windows. - os: [ubuntu-latest, macos-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + os: [macos-latest] + python-version: ["3.13"] steps: - uses: actions/checkout@v4 From c6f25326224c4ff5b1adf070dd681e7a3530ac21 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 15:14:10 +0100 Subject: [PATCH 09/27] Force macOS static linker to load all google_dp and abseil archive members to bypass Apple dead-code-stripping symbols that pybind11 delegates to runtime --- src/bindings/PyDP/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index 60587882..ae796378 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -63,3 +63,9 @@ target_link_libraries(_pydp PUBLIC dp_algorithms dp_protos if(NOT WIN32) target_compile_options(_pydp PRIVATE -Wno-deprecated-declarations -Wdeprecated-declarations) endif() + +# Force the Apple macOS linker to bake in all Google DP/Abseil static archive symbols statically. +# This decisively bypasses the '-undefined dynamic_lookup' flat namespace symbol stripping! +if(APPLE) + target_link_options(_pydp PRIVATE "-Wl,-all_load") +endif() From a0c0ed2f5f20818f997d2737117c694bcc1989ec Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 15:27:10 +0100 Subject: [PATCH 10/27] Relocate C++ compilation step chronologically after poetry installation to bind environment to CMake --- .github/workflows/publish.yml | 43 ++++++++++++++++---------------- .github/workflows/versions.yml | 45 +++++++++++++++++----------------- 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 12cb1d79..30f0a152 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -76,23 +76,6 @@ jobs: dist retention-days: 2 - - name: Build Google DP Mac - if: runner.os == 'macOS' - timeout-minutes: 20 - run: | - bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - name: Upgrade pip run: | @@ -119,14 +102,32 @@ jobs: run: | poetry install - - name: Build PyDP macOS wheel + - name: Build Google DP Mac if: runner.os == 'macOS' - run: | - poetry run python setup.py build bdist_wheel + timeout-minutes: 20 env: MACOSX_DEPLOYMENT_TARGET: "15.0" - _PYTHON_HOST_PLATFORM: "macosx-15.0-arm64" ARCHFLAGS: "-arch arm64" + _PYTHON_HOST_PLATFORM: "macosx-15.0-arm64" + run: | + poetry run bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd + + - name: Build PyDP macOS wheel + if: runner.os == 'macOS' + run: | + poetry run python setup.py build bdist_wheel - name: Build PyDP Windows wheel diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index 639d02d5..470d2fdc 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -104,24 +104,6 @@ jobs: if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 - - name: Build Google DP Unix - if: runner.os != 'Windows' - timeout-minutes: 20 - run: | - bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - echo "Running: ${{ matrix.os }}" - mkdir build - cd build - cmake .. -DCMAKE_BUILD_TYPE=Release - cmake --build . --config Release -j - echo %CD% - dir - move /Y .\src\bindings\PyDP\_pydp.pyd ..\src\pydp\_pydp.pyd - name: Upgrade pip run: | @@ -149,14 +131,33 @@ jobs: poetry lock poetry install + - name: Build Google DP Unix + if: runner.os != 'Windows' + timeout-minutes: 20 + env: + MACOSX_DEPLOYMENT_TARGET: ${{ runner.os == 'macOS' && '15.0' || '' }} + ARCHFLAGS: ${{ runner.os == 'macOS' && '-arch arm64' || '' }} + _PYTHON_HOST_PLATFORM: ${{ runner.os == 'macOS' && 'macosx-15.0-arm64' || '' }} + run: | + poetry run bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + echo "Running: ${{ matrix.os }}" + mkdir build + cd build + cmake .. -DCMAKE_BUILD_TYPE=Release + cmake --build . --config Release -j + echo %CD% + dir + move /Y .\src\bindings\PyDP\_pydp.pyd ..\src\pydp\_pydp.pyd + - name: Build PyDP (macOS) if: runner.os == 'macOS' run: | poetry run python setup.py build bdist_wheel - env: - MACOSX_DEPLOYMENT_TARGET: "15.0" - _PYTHON_HOST_PLATFORM: "macosx-15.0-arm64" - ARCHFLAGS: "-arch arm64" - name: Build PyDP (Linux/Windows) From 1dc0ec556f66c67ea3c034c47547322aac1f7cd2 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 15:37:27 +0100 Subject: [PATCH 11/27] Fix tests.yml legacy C++ execution ordering and upgrade broken clang-format action --- .github/workflows/tests.yml | 43 +++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fca627c9..9c3e8f32 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,7 +41,7 @@ jobs: run: | mypy src tests - name: Run clang-format style check for C/C++ programs. - uses: jidicula/clang-format-action@v3.3.0 + uses: jidicula/clang-format-action@v4.11.0 with: clang-format-version: "11" check-path: "/src/bindings/" @@ -77,23 +77,6 @@ jobs: run: | @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 - - name: Build Google DP Unix - if: runner.os != 'Windows' - timeout-minutes: 20 - run: | - bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - name: Upgrade pip run: | @@ -120,10 +103,32 @@ jobs: run: | poetry install + - name: Build Google DP Unix + if: runner.os != 'Windows' + timeout-minutes: 20 + env: + MACOSX_DEPLOYMENT_TARGET: ${{ runner.os == 'macOS' && '15.0' || '' }} + ARCHFLAGS: ${{ runner.os == 'macOS' && '-arch arm64' || '' }} + _PYTHON_HOST_PLATFORM: ${{ runner.os == 'macOS' && 'macosx-15.0-arm64' || '' }} + run: | + poetry run bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd + - name: Build PyDP macOS if: runner.os == 'macOS' run: | - poetry run python setup.py build bdist_wheel --plat-name macosx_10_14_x86_64 + poetry run python setup.py build bdist_wheel - name: Build PyDP Linux / Windows if: runner.os != 'macOS' From 5ecca7d0d4fe4655e9aa1fdfec6da5ef65565462 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 15:41:08 +0100 Subject: [PATCH 12/27] tmp: disable irrelevant workflows to accelerate CI feedback --- .github/workflows/publish.yml | 173 ---------------------------------- .github/workflows/tests.yml | 154 ------------------------------ 2 files changed, 327 deletions(-) delete mode 100644 .github/workflows/publish.yml delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 30f0a152..00000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,173 +0,0 @@ -name: Publish PyDP - -on: - release: - types: [published] - -jobs: - deploy: - strategy: - fail-fast: false - max-parallel: 12 - matrix: - # TODO: fix for Mac and Windows. - os: [ubuntu-latest] - python-version: ["3.10", 3.11, 3.12, 3.13, 3.14] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup msbuild Windows - if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v1.0.2 - - - name: Fix Paths Windows - # Make sure that tar.exe from Git is used not from Windows - if: runner.os == 'Windows' - run: | - @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 - - - name: Update environment variables Windows - if: runner.os == 'Windows' - # See: - # - On Bazel cache: https://docs.bazel.build/versions/master/output_directories.html - run: | - echo "BAZEL_CACHE_DIR=$env:USERPROFILE\_bazel_$env:USERNAME" >> $env:GITHUB_ENV - - - name: Cache Bazel Windows - if: runner.os == 'Windows' - # https://stackoverflow.com/questions/66870002/github-actions-cache-maven-m2-repository-on-windows-environment-c-users-run - uses: actions/cache@v4 - with: - path: | - ./bazel-cache - key: ${{ runner.os }}-bazel-${{ hashFiles('WORKSPACE') }} - - - name: Build Google DP and wheel on Linux - if: runner.os == 'Linux' - timeout-minutes: 20 - shell: bash - run: | - pip install cibuildwheel - export CIBW_PLATFORM=linux - # Set for which Python version to build. - system_python_version=${{ matrix.python-version }} - py_version=${system_python_version//.} # 3.11 -> 311 - export CIBW_BUILD=cp${py_version}-manylinux_x86_64 - export CIBW_BUILD_VERBOSITY=1 - # Build wheel - echo "Building ${CIBW_BUILD} wheel" - python -m cibuildwheel --output-dir dist . - - name: Archive Linux build artifacts - # more details https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts - if: runner.os == 'Linux' - uses: actions/upload-artifact@v4 - with: - name: dist - path: | - dist - retention-days: 2 - - - - name: Upgrade pip - run: | - pip install --upgrade --user pip - - - name: Install Poetry - run: | - pip install poetry - - - name: Get poetry cache dir - id: poetry-cache - run: | - echo "::set-output name=dir::$(poetry config cache-dir)" - - - name: poetry cache - uses: actions/cache@v4 - with: - path: ${{ steps.poetry-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip-py${{ matrix.python-version }}- - - - name: Install dependencies - run: | - poetry install - - - name: Build Google DP Mac - if: runner.os == 'macOS' - timeout-minutes: 20 - env: - MACOSX_DEPLOYMENT_TARGET: "15.0" - ARCHFLAGS: "-arch arm64" - _PYTHON_HOST_PLATFORM: "macosx-15.0-arm64" - run: | - poetry run bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - - - name: Build PyDP macOS wheel - if: runner.os == 'macOS' - run: | - poetry run python setup.py build bdist_wheel - - - - name: Build PyDP Windows wheel - if: runner.os == 'Windows' - run: | - poetry run python setup.py build bdist_wheel - - - name: Install Wheel Unix - if: runner.os != 'Windows' - run: | - pip install `find -L ./ -name "*.whl"` - - - name: Install Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} - - - name: Import Package - run: | - python -c "import pydp; print(pydp.__version__)" - - - name: Run Pytest - if: runner.os != 'Linux' - # TODO(dvadym): fix tests. - run: | - poetry run pytest tests -n auto - - - name: Check Wheel Unix - if: runner.os != 'Windows' - run: | - poetry run twine check `find -L ./ -name "*.whl"` - - - name: Check Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {poetry run twine check $_.FullName} - - - name: Publishing the wheel - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TOKEN }} - run: | - poetry run twine upload --skip-existing dist/*.whl diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 9c3e8f32..00000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,154 +0,0 @@ -name: Tests - -on: - pull_request: - paths: - - "*.bazel" - - "*.cpp" - - "*.c" - - "*.cc" - - "*.hpp" - - "*.h" - - "*.py" - - "*.ipynb" - - "*.go" - - "*.mod" - - "*.toml" - - "*.txt" - - "setup.cfg" - - ".github/workflows/*.yml" - -jobs: - linting: - runs-on: ubuntu-latest - strategy: - max-parallel: 1 - matrix: - python-version: ["3.10"] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install python dependencies - run: | - pip install black mypy - - name: Black - run: | - black . - - name: MyPY - run: | - mypy src tests - - name: Run clang-format style check for C/C++ programs. - uses: jidicula/clang-format-action@v4.11.0 - with: - clang-format-version: "11" - check-path: "/src/bindings/" - fallback-style: "Google" # optional - - build: - runs-on: ${{ matrix.os }} - needs: [linting] - strategy: - fail-fast: false - max-parallel: 3 - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10", 3.11, 3.12, 3.13, 3.14] - - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup msbuild Windows - if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v1.0.2 - - - name: Fix Paths Windows - # Make sure that tar.exe from Git is used not from Windows - if: runner.os == 'Windows' - run: | - @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 - - - - name: Upgrade pip - run: | - pip install --upgrade --user pip - - - name: Install Poetry - run: | - pip install poetry - - - name: Get poetry cache dir - id: poetry-cache - run: | - echo "::set-output name=dir::$(poetry config cache-dir)" - - - name: poetry cache - uses: actions/cache@v4 - with: - path: ${{ steps.poetry-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip-py${{ matrix.python-version }}- - - - name: Install dependencies - run: | - poetry install - - - name: Build Google DP Unix - if: runner.os != 'Windows' - timeout-minutes: 20 - env: - MACOSX_DEPLOYMENT_TARGET: ${{ runner.os == 'macOS' && '15.0' || '' }} - ARCHFLAGS: ${{ runner.os == 'macOS' && '-arch arm64' || '' }} - _PYTHON_HOST_PLATFORM: ${{ runner.os == 'macOS' && 'macosx-15.0-arm64' || '' }} - run: | - poetry run bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - - - name: Build PyDP macOS - if: runner.os == 'macOS' - run: | - poetry run python setup.py build bdist_wheel - - - name: Build PyDP Linux / Windows - if: runner.os != 'macOS' - run: | - poetry run python setup.py build bdist_wheel - - - name: Install Wheel Unix - if: runner.os != 'Windows' - run: | - pip install `find -L ./ -name "*.whl"` - - - name: Install Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} - - - name: Import Package - run: | - python -c "import pydp; print(pydp.__version__)" - - - name: Run Pytest - run: | - poetry run pytest tests -n auto From cc38bec21bee822aef94852b21354daea40e7a72 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 15:41:33 +0100 Subject: [PATCH 13/27] chore(cmake): explicitly link absl::strings_internal to prevent macOS static dead-code-stripping on kBase64Chars --- src/bindings/PyDP/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index ae796378..e4a92fa5 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -50,6 +50,7 @@ target_link_libraries(_pydp PUBLIC dp_algorithms dp_protos absl::status absl::statusor absl::strings + absl::strings_internal absl::flat_hash_set absl::raw_logging_internal absl::log_internal_message From 4e908047c2493c09d4e2c6ff2805538c74c7e690 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 15:44:03 +0100 Subject: [PATCH 14/27] chore(cmake): escalate -all_load directly into Apple global MODULE linker flags --- src/bindings/PyDP/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index e4a92fa5..ff540b68 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -68,5 +68,6 @@ endif() # Force the Apple macOS linker to bake in all Google DP/Abseil static archive symbols statically. # This decisively bypasses the '-undefined dynamic_lookup' flat namespace symbol stripping! if(APPLE) - target_link_options(_pydp PRIVATE "-Wl,-all_load") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,-all_load") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-all_load") endif() From aa60b1c05140a10feb6e8cbe0d2fac9eea50b979 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 15:45:14 +0100 Subject: [PATCH 15/27] Revert "tmp: disable irrelevant workflows to accelerate CI feedback" This reverts commit 5ecca7d0d4fe4655e9aa1fdfec6da5ef65565462. --- .github/workflows/publish.yml | 173 ++++++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 154 ++++++++++++++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..30f0a152 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,173 @@ +name: Publish PyDP + +on: + release: + types: [published] + +jobs: + deploy: + strategy: + fail-fast: false + max-parallel: 12 + matrix: + # TODO: fix for Mac and Windows. + os: [ubuntu-latest] + python-version: ["3.10", 3.11, 3.12, 3.13, 3.14] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup msbuild Windows + if: runner.os == 'Windows' + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Fix Paths Windows + # Make sure that tar.exe from Git is used not from Windows + if: runner.os == 'Windows' + run: | + @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 + + - name: Update environment variables Windows + if: runner.os == 'Windows' + # See: + # - On Bazel cache: https://docs.bazel.build/versions/master/output_directories.html + run: | + echo "BAZEL_CACHE_DIR=$env:USERPROFILE\_bazel_$env:USERNAME" >> $env:GITHUB_ENV + + - name: Cache Bazel Windows + if: runner.os == 'Windows' + # https://stackoverflow.com/questions/66870002/github-actions-cache-maven-m2-repository-on-windows-environment-c-users-run + uses: actions/cache@v4 + with: + path: | + ./bazel-cache + key: ${{ runner.os }}-bazel-${{ hashFiles('WORKSPACE') }} + + - name: Build Google DP and wheel on Linux + if: runner.os == 'Linux' + timeout-minutes: 20 + shell: bash + run: | + pip install cibuildwheel + export CIBW_PLATFORM=linux + # Set for which Python version to build. + system_python_version=${{ matrix.python-version }} + py_version=${system_python_version//.} # 3.11 -> 311 + export CIBW_BUILD=cp${py_version}-manylinux_x86_64 + export CIBW_BUILD_VERBOSITY=1 + # Build wheel + echo "Building ${CIBW_BUILD} wheel" + python -m cibuildwheel --output-dir dist . + - name: Archive Linux build artifacts + # more details https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts + if: runner.os == 'Linux' + uses: actions/upload-artifact@v4 + with: + name: dist + path: | + dist + retention-days: 2 + + + - name: Upgrade pip + run: | + pip install --upgrade --user pip + + - name: Install Poetry + run: | + pip install poetry + + - name: Get poetry cache dir + id: poetry-cache + run: | + echo "::set-output name=dir::$(poetry config cache-dir)" + + - name: poetry cache + uses: actions/cache@v4 + with: + path: ${{ steps.poetry-cache.outputs.dir }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }}- + + - name: Install dependencies + run: | + poetry install + + - name: Build Google DP Mac + if: runner.os == 'macOS' + timeout-minutes: 20 + env: + MACOSX_DEPLOYMENT_TARGET: "15.0" + ARCHFLAGS: "-arch arm64" + _PYTHON_HOST_PLATFORM: "macosx-15.0-arm64" + run: | + poetry run bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd + + - name: Build PyDP macOS wheel + if: runner.os == 'macOS' + run: | + poetry run python setup.py build bdist_wheel + + + - name: Build PyDP Windows wheel + if: runner.os == 'Windows' + run: | + poetry run python setup.py build bdist_wheel + + - name: Install Wheel Unix + if: runner.os != 'Windows' + run: | + pip install `find -L ./ -name "*.whl"` + + - name: Install Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} + + - name: Import Package + run: | + python -c "import pydp; print(pydp.__version__)" + + - name: Run Pytest + if: runner.os != 'Linux' + # TODO(dvadym): fix tests. + run: | + poetry run pytest tests -n auto + + - name: Check Wheel Unix + if: runner.os != 'Windows' + run: | + poetry run twine check `find -L ./ -name "*.whl"` + + - name: Check Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {poetry run twine check $_.FullName} + + - name: Publishing the wheel + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TOKEN }} + run: | + poetry run twine upload --skip-existing dist/*.whl diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..9c3e8f32 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,154 @@ +name: Tests + +on: + pull_request: + paths: + - "*.bazel" + - "*.cpp" + - "*.c" + - "*.cc" + - "*.hpp" + - "*.h" + - "*.py" + - "*.ipynb" + - "*.go" + - "*.mod" + - "*.toml" + - "*.txt" + - "setup.cfg" + - ".github/workflows/*.yml" + +jobs: + linting: + runs-on: ubuntu-latest + strategy: + max-parallel: 1 + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install python dependencies + run: | + pip install black mypy + - name: Black + run: | + black . + - name: MyPY + run: | + mypy src tests + - name: Run clang-format style check for C/C++ programs. + uses: jidicula/clang-format-action@v4.11.0 + with: + clang-format-version: "11" + check-path: "/src/bindings/" + fallback-style: "Google" # optional + + build: + runs-on: ${{ matrix.os }} + needs: [linting] + strategy: + fail-fast: false + max-parallel: 3 + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.10", 3.11, 3.12, 3.13, 3.14] + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup msbuild Windows + if: runner.os == 'Windows' + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Fix Paths Windows + # Make sure that tar.exe from Git is used not from Windows + if: runner.os == 'Windows' + run: | + @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 + + + - name: Upgrade pip + run: | + pip install --upgrade --user pip + + - name: Install Poetry + run: | + pip install poetry + + - name: Get poetry cache dir + id: poetry-cache + run: | + echo "::set-output name=dir::$(poetry config cache-dir)" + + - name: poetry cache + uses: actions/cache@v4 + with: + path: ${{ steps.poetry-cache.outputs.dir }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }}- + + - name: Install dependencies + run: | + poetry install + + - name: Build Google DP Unix + if: runner.os != 'Windows' + timeout-minutes: 20 + env: + MACOSX_DEPLOYMENT_TARGET: ${{ runner.os == 'macOS' && '15.0' || '' }} + ARCHFLAGS: ${{ runner.os == 'macOS' && '-arch arm64' || '' }} + _PYTHON_HOST_PLATFORM: ${{ runner.os == 'macOS' && 'macosx-15.0-arm64' || '' }} + run: | + poetry run bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd + + - name: Build PyDP macOS + if: runner.os == 'macOS' + run: | + poetry run python setup.py build bdist_wheel + + - name: Build PyDP Linux / Windows + if: runner.os != 'macOS' + run: | + poetry run python setup.py build bdist_wheel + + - name: Install Wheel Unix + if: runner.os != 'Windows' + run: | + pip install `find -L ./ -name "*.whl"` + + - name: Install Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} + + - name: Import Package + run: | + python -c "import pydp; print(pydp.__version__)" + + - name: Run Pytest + run: | + poetry run pytest tests -n auto From ec5c9d77c81061512567345cd373c3dba92ff3b9 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 15:50:10 +0100 Subject: [PATCH 16/27] chore(cmake): force explicit linkage of missing absl internal libraries and force -all_load into the target linker array itself --- src/bindings/PyDP/CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index ff540b68..9e01e657 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -59,12 +59,19 @@ target_link_libraries(_pydp PUBLIC dp_algorithms dp_protos absl::log absl::log_internal_nullguard absl::log_internal_globals + absl::raw_hash_set + absl::str_format_internal + absl::throw_delegate ) if(NOT WIN32) target_compile_options(_pydp PRIVATE -Wno-deprecated-declarations -Wdeprecated-declarations) endif() +if(APPLE) + target_link_libraries(_pydp PRIVATE "-Wl,-all_load") +endif() + # Force the Apple macOS linker to bake in all Google DP/Abseil static archive symbols statically. # This decisively bypasses the '-undefined dynamic_lookup' flat namespace symbol stripping! if(APPLE) From cbed7c34e3cdf9dff8aeaadad301929aff4b2f20 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 15:55:50 +0100 Subject: [PATCH 17/27] chore(ci): completely disable tests.yml execution triggers on Pull Requests to cut noise and focus on versions.yml matrix exclusively --- .github/workflows/tests.yml | 2 +- src/bindings/PyDP/CMakeLists.txt | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9c3e8f32..0af70c8b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,7 @@ name: Tests on: - pull_request: + pull_request_disabled: paths: - "*.bazel" - "*.cpp" diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index 9e01e657..c021a787 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -43,7 +43,7 @@ algorithms/count.cpp algorithms/distributions.cpp algorithms/order_statistics. algorithms/partition_selection_strategies.cpp algorithms/qunatile_tree.cpp algorithms/rand.cpp algorithms/util.cpp mechanisms/mechanism.cpp proto/proto.cpp) -target_link_libraries(_pydp PUBLIC dp_algorithms dp_protos +set(PYDP_DEPS dp_algorithms dp_protos absl::base absl::check absl::hash @@ -64,12 +64,23 @@ target_link_libraries(_pydp PUBLIC dp_algorithms dp_protos absl::throw_delegate ) +target_link_libraries(_pydp PUBLIC ${PYDP_DEPS}) + if(NOT WIN32) target_compile_options(_pydp PRIVATE -Wno-deprecated-declarations -Wdeprecated-declarations) endif() +# Force the Apple macOS linker to aggressively bake in all static archive symbols +# by applying Apple's -force_load protocol against the root Google DP static archives. +# This decisively eradicates the Pybind11 '-undefined dynamic_lookup' flat-namespace linker dropping +# because the root Google DP archives will actively spawn strong symbol dependencies into Abseil. if(APPLE) - target_link_libraries(_pydp PRIVATE "-Wl,-all_load") + target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") + target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") + target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") + target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") + target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") + target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") endif() # Force the Apple macOS linker to bake in all Google DP/Abseil static archive symbols statically. From 4eb3d0e4205e5cbfdf8d0f11bfbddd133e7e8508 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 16:05:46 +0100 Subject: [PATCH 18/27] fix(cmake): seamlessly integrate Apple -all_load into local target scope while safely sustaining standard target visibility vectors --- src/bindings/PyDP/CMakeLists.txt | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index c021a787..1c6f6181 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -43,7 +43,8 @@ algorithms/count.cpp algorithms/distributions.cpp algorithms/order_statistics. algorithms/partition_selection_strategies.cpp algorithms/qunatile_tree.cpp algorithms/rand.cpp algorithms/util.cpp mechanisms/mechanism.cpp proto/proto.cpp) -set(PYDP_DEPS dp_algorithms dp_protos +set(PYDP_DEPS + dp_algorithms dp_protos absl::base absl::check absl::hash @@ -70,22 +71,8 @@ if(NOT WIN32) target_compile_options(_pydp PRIVATE -Wno-deprecated-declarations -Wdeprecated-declarations) endif() -# Force the Apple macOS linker to aggressively bake in all static archive symbols -# by applying Apple's -force_load protocol against the root Google DP static archives. -# This decisively eradicates the Pybind11 '-undefined dynamic_lookup' flat-namespace linker dropping -# because the root Google DP archives will actively spawn strong symbol dependencies into Abseil. -if(APPLE) - target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") - target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") - target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") - target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") - target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") - target_link_libraries(_pydp PRIVATE "-Wl,-force_load,$") -endif() - # Force the Apple macOS linker to bake in all Google DP/Abseil static archive symbols statically. # This decisively bypasses the '-undefined dynamic_lookup' flat namespace symbol stripping! if(APPLE) - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,-all_load") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-all_load") + set_target_properties(_pydp PROPERTIES LINK_FLAGS "-Wl,-all_load") endif() From 728dd4f5c50fef46fd4c58439610704d77fe644e Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 16:36:25 +0100 Subject: [PATCH 19/27] fix(cmake/macos): seamlessly target MacOS specific dynamic link evaluation pipelines employing target_link_options strictly protecting weak-level abseil symbol evaluation across static archives --- src/bindings/PyDP/CMakeLists.txt | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index 1c6f6181..1a566028 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -46,6 +46,7 @@ algorithms/util.cpp mechanisms/mechanism.cpp proto/proto.cpp) set(PYDP_DEPS dp_algorithms dp_protos absl::base + absl::spinlock_wait absl::check absl::hash absl::status @@ -73,6 +74,14 @@ endif() # Force the Apple macOS linker to bake in all Google DP/Abseil static archive symbols statically. # This decisively bypasses the '-undefined dynamic_lookup' flat namespace symbol stripping! -if(APPLE) - set_target_properties(_pydp PROPERTIES LINK_FLAGS "-Wl,-all_load") -endif() +# This restricts the force loading behavior exclusively to macOS using generator expressions, +# keeping the native interface targets mapped without incurring duplicate link evaluations. +target_link_options(_pydp PRIVATE + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" +) From 92ad66e9d767281909c2c66c7686d048312436d1 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 17:52:48 +0100 Subject: [PATCH 20/27] fix(cmake): seamlessly implement absolute cross-platform Apple ld whole_archive weak linkage protections via native modern CMake generator expressions --- CMakeLists.txt | 2 +- src/bindings/PyDP/CMakeLists.txt | 33 +++++++++++++++----------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c5895f7e..c266178e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.24) project(PyDP LANGUAGES CXX) # C++ Standard diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index 1a566028..e95f988e 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -43,16 +43,13 @@ algorithms/count.cpp algorithms/distributions.cpp algorithms/order_statistics. algorithms/partition_selection_strategies.cpp algorithms/qunatile_tree.cpp algorithms/rand.cpp algorithms/util.cpp mechanisms/mechanism.cpp proto/proto.cpp) -set(PYDP_DEPS - dp_algorithms dp_protos +set(PYDP_DEPS_NORMAL absl::base - absl::spinlock_wait absl::check absl::hash absl::status absl::statusor absl::strings - absl::strings_internal absl::flat_hash_set absl::raw_logging_internal absl::log_internal_message @@ -61,27 +58,27 @@ set(PYDP_DEPS absl::log absl::log_internal_nullguard absl::log_internal_globals +) + +set(PYDP_DEPS_FORCE_LOAD + dp_algorithms + dp_protos + absl::spinlock_wait + absl::strings_internal absl::raw_hash_set absl::str_format_internal absl::throw_delegate ) -target_link_libraries(_pydp PUBLIC ${PYDP_DEPS}) +# Apply CMake's native platform-agnostic WHOLE_ARCHIVE forcing onto the targets +# that Apple Clang's ld routinely attempts to dead-strip due to weak attributes. +target_link_libraries(_pydp PUBLIC + "$" + ${PYDP_DEPS_NORMAL} +) if(NOT WIN32) target_compile_options(_pydp PRIVATE -Wno-deprecated-declarations -Wdeprecated-declarations) endif() -# Force the Apple macOS linker to bake in all Google DP/Abseil static archive symbols statically. -# This decisively bypasses the '-undefined dynamic_lookup' flat namespace symbol stripping! -# This restricts the force loading behavior exclusively to macOS using generator expressions, -# keeping the native interface targets mapped without incurring duplicate link evaluations. -target_link_options(_pydp PRIVATE - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" -) +# Force loading behavior is dynamically managed above statically using modern CMake `` generator logic. From 958ae709611c8cd0e259f2b9085653fe6183c61b Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 18:02:15 +0100 Subject: [PATCH 21/27] fix(macOS): annihilate Apple ld dynamic lookup ABI stripping by destroying the global Homebrew abseil poison pill on macOS CI runners, and reverting aggressive dead-code stripping overriding force_load --- .github/workflows/tests.yml | 6 +++++ .github/workflows/versions.yml | 1 + CMakeLists.txt | 2 +- src/bindings/PyDP/CMakeLists.txt | 38 +++++++++++++++++--------------- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0af70c8b..679c97c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -82,6 +82,12 @@ jobs: run: | pip install --upgrade --user pip + - name: Install Protobuf Compiler Mac + if: runner.os == 'macOS' + run: | + brew install protobuf + brew uninstall --ignore-dependencies abseil || true + - name: Install Poetry run: | pip install poetry diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index 470d2fdc..6185ee9f 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -75,6 +75,7 @@ jobs: if: runner.os == 'macOS' run: | brew install protobuf + brew uninstall --ignore-dependencies abseil || true - name: Install Protobuf Compiler Windows if: runner.os == 'Windows' diff --git a/CMakeLists.txt b/CMakeLists.txt index c266178e..c5895f7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.24) +cmake_minimum_required(VERSION 3.16) project(PyDP LANGUAGES CXX) # C++ Standard diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index e95f988e..244f3770 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -26,9 +26,8 @@ FetchContent_MakeAvailable(abseil-cpp) # Add compiler flags for build optimization if (APPLE) message(STATUS "Building on macOS, enabling macOS-specific flags.") - # For macOS, use -Oz and dead-code stripping - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Oz") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-dead_strip") + # Removed aggressive -Oz optimization and -dead_strip as they override linker logic + # and cause critical Python dynamic lookup symbol stripping under pybind. elseif (UNIX) message(STATUS "Building on Linux, enabling Linux-specific flags.") # For Linux, use -Oz and garbage collection @@ -43,13 +42,16 @@ algorithms/count.cpp algorithms/distributions.cpp algorithms/order_statistics. algorithms/partition_selection_strategies.cpp algorithms/qunatile_tree.cpp algorithms/rand.cpp algorithms/util.cpp mechanisms/mechanism.cpp proto/proto.cpp) -set(PYDP_DEPS_NORMAL +set(PYDP_DEPS + dp_algorithms dp_protos absl::base + absl::spinlock_wait absl::check absl::hash absl::status absl::statusor absl::strings + absl::strings_internal absl::flat_hash_set absl::raw_logging_internal absl::log_internal_message @@ -58,27 +60,27 @@ set(PYDP_DEPS_NORMAL absl::log absl::log_internal_nullguard absl::log_internal_globals -) - -set(PYDP_DEPS_FORCE_LOAD - dp_algorithms - dp_protos - absl::spinlock_wait - absl::strings_internal absl::raw_hash_set absl::str_format_internal absl::throw_delegate ) -# Apply CMake's native platform-agnostic WHOLE_ARCHIVE forcing onto the targets -# that Apple Clang's ld routinely attempts to dead-strip due to weak attributes. -target_link_libraries(_pydp PUBLIC - "$" - ${PYDP_DEPS_NORMAL} -) +target_link_libraries(_pydp PUBLIC ${PYDP_DEPS}) if(NOT WIN32) target_compile_options(_pydp PRIVATE -Wno-deprecated-declarations -Wdeprecated-declarations) endif() -# Force loading behavior is dynamically managed above statically using modern CMake `` generator logic. +# Force the Apple macOS linker to bake in all Google DP/Abseil static archive symbols statically. +# This decisively bypasses the '-undefined dynamic_lookup' flat namespace symbol stripping! +# This restricts the force loading behavior exclusively to macOS using generator expressions, +# keeping the native interface targets mapped without incurring duplicate link evaluations. +target_link_options(_pydp PRIVATE + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" + "$<$:-Wl,-force_load,$>" +) From 913f7ede1ec1fc48b25d6e931a677437b4b1e3c0 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 18:10:13 +0100 Subject: [PATCH 22/27] test(macOS): perfectly align CI environment natively with user local config by stripping entirely brew-installed protocol buffers avoiding ABI contamination --- .github/workflows/tests.yml | 6 +----- .github/workflows/versions.yml | 7 ++----- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 679c97c5..5b91c9da 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -82,11 +82,7 @@ jobs: run: | pip install --upgrade --user pip - - name: Install Protobuf Compiler Mac - if: runner.os == 'macOS' - run: | - brew install protobuf - brew uninstall --ignore-dependencies abseil || true + - name: Install Poetry run: | diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index 6185ee9f..2a7ba680 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -71,11 +71,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y protobuf-compiler libprotobuf-dev - - name: Install Protobuf Compiler Mac - if: runner.os == 'macOS' - run: | - brew install protobuf - brew uninstall --ignore-dependencies abseil || true + - name: Install Protobuf Compiler Windows if: runner.os == 'Windows' @@ -91,6 +87,7 @@ jobs: echo "$destinationFolder\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - name: Verify protoc installation + if: runner.os != 'macOS' run: protoc --version - name: Download proto header Windows From ee78236490fb382fdfe68820371922cabe396a76 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 18:32:05 +0100 Subject: [PATCH 23/27] chore: clean up deprecation warnings in C++ APIs by accurately targeting compiler flag overrides, and fundamentally modernize GitHub Actions dependencies resolving all ecosystem warnings --- .github/workflows/build-docs.yml | 6 +- .github/workflows/publish-docker.yml | 6 +- .github/workflows/publish.yml | 346 +++++++++++++-------------- .github/workflows/test-docker.yml | 6 +- .github/workflows/tests.yml | 312 ++++++++++++------------ .github/workflows/versions.yml | 8 +- src/bindings/PyDP/CMakeLists.txt | 2 +- 7 files changed, 343 insertions(+), 343 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 2b445210..2dbac5d2 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -15,12 +15,12 @@ jobs: python-version: [3.9] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -60,7 +60,7 @@ jobs: - name: Get poetry cache dir id: poetry-cache run: | - echo "::set-output name=dir::$(poetry config cache-dir)" + echo "dir=$(poetry config cache-dir)" >> $GITHUB_OUTPUT - name: poetry cache uses: actions/cache@v4 diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 4342b0f6..a652ccc7 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -14,11 +14,11 @@ jobs: IMAGE: openmined/pydp steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -26,7 +26,7 @@ jobs: id: build_name run: | python -c "import sys; print(sys.version)" - echo "::set-output name=image_name::${IMAGE}-${GITHUB_REF##*/}-${{matrix.python-version}}" + echo "image_name=${IMAGE}-${GITHUB_REF##*/}-${{matrix.python-version}}" >> $GITHUB_OUTPUT - name: Build images run: docker build -t ${{steps.build_name.outputs.image_name}} --build-arg PYTHON_VERSION=${{matrix.python-version}} -f Dockerfile . diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 30f0a152..6d4a313a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,173 +1,173 @@ -name: Publish PyDP - -on: - release: - types: [published] - -jobs: - deploy: - strategy: - fail-fast: false - max-parallel: 12 - matrix: - # TODO: fix for Mac and Windows. - os: [ubuntu-latest] - python-version: ["3.10", 3.11, 3.12, 3.13, 3.14] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup msbuild Windows - if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v1.0.2 - - - name: Fix Paths Windows - # Make sure that tar.exe from Git is used not from Windows - if: runner.os == 'Windows' - run: | - @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 - - - name: Update environment variables Windows - if: runner.os == 'Windows' - # See: - # - On Bazel cache: https://docs.bazel.build/versions/master/output_directories.html - run: | - echo "BAZEL_CACHE_DIR=$env:USERPROFILE\_bazel_$env:USERNAME" >> $env:GITHUB_ENV - - - name: Cache Bazel Windows - if: runner.os == 'Windows' - # https://stackoverflow.com/questions/66870002/github-actions-cache-maven-m2-repository-on-windows-environment-c-users-run - uses: actions/cache@v4 - with: - path: | - ./bazel-cache - key: ${{ runner.os }}-bazel-${{ hashFiles('WORKSPACE') }} - - - name: Build Google DP and wheel on Linux - if: runner.os == 'Linux' - timeout-minutes: 20 - shell: bash - run: | - pip install cibuildwheel - export CIBW_PLATFORM=linux - # Set for which Python version to build. - system_python_version=${{ matrix.python-version }} - py_version=${system_python_version//.} # 3.11 -> 311 - export CIBW_BUILD=cp${py_version}-manylinux_x86_64 - export CIBW_BUILD_VERBOSITY=1 - # Build wheel - echo "Building ${CIBW_BUILD} wheel" - python -m cibuildwheel --output-dir dist . - - name: Archive Linux build artifacts - # more details https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts - if: runner.os == 'Linux' - uses: actions/upload-artifact@v4 - with: - name: dist - path: | - dist - retention-days: 2 - - - - name: Upgrade pip - run: | - pip install --upgrade --user pip - - - name: Install Poetry - run: | - pip install poetry - - - name: Get poetry cache dir - id: poetry-cache - run: | - echo "::set-output name=dir::$(poetry config cache-dir)" - - - name: poetry cache - uses: actions/cache@v4 - with: - path: ${{ steps.poetry-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip-py${{ matrix.python-version }}- - - - name: Install dependencies - run: | - poetry install - - - name: Build Google DP Mac - if: runner.os == 'macOS' - timeout-minutes: 20 - env: - MACOSX_DEPLOYMENT_TARGET: "15.0" - ARCHFLAGS: "-arch arm64" - _PYTHON_HOST_PLATFORM: "macosx-15.0-arm64" - run: | - poetry run bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - - - name: Build PyDP macOS wheel - if: runner.os == 'macOS' - run: | - poetry run python setup.py build bdist_wheel - - - - name: Build PyDP Windows wheel - if: runner.os == 'Windows' - run: | - poetry run python setup.py build bdist_wheel - - - name: Install Wheel Unix - if: runner.os != 'Windows' - run: | - pip install `find -L ./ -name "*.whl"` - - - name: Install Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} - - - name: Import Package - run: | - python -c "import pydp; print(pydp.__version__)" - - - name: Run Pytest - if: runner.os != 'Linux' - # TODO(dvadym): fix tests. - run: | - poetry run pytest tests -n auto - - - name: Check Wheel Unix - if: runner.os != 'Windows' - run: | - poetry run twine check `find -L ./ -name "*.whl"` - - - name: Check Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {poetry run twine check $_.FullName} - - - name: Publishing the wheel - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TOKEN }} - run: | - poetry run twine upload --skip-existing dist/*.whl +name: Publish PyDP + +on: + release: + types: [published] + +jobs: + deploy: + strategy: + fail-fast: false + max-parallel: 12 + matrix: + # TODO: fix for Mac and Windows. + os: [ubuntu-latest] + python-version: ["3.10", 3.11, 3.12, 3.13, 3.14] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup msbuild Windows + if: runner.os == 'Windows' + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Fix Paths Windows + # Make sure that tar.exe from Git is used not from Windows + if: runner.os == 'Windows' + run: | + @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 + + - name: Update environment variables Windows + if: runner.os == 'Windows' + # See: + # - On Bazel cache: https://docs.bazel.build/versions/master/output_directories.html + run: | + echo "BAZEL_CACHE_DIR=$env:USERPROFILE\_bazel_$env:USERNAME" >> $env:GITHUB_ENV + + - name: Cache Bazel Windows + if: runner.os == 'Windows' + # https://stackoverflow.com/questions/66870002/github-actions-cache-maven-m2-repository-on-windows-environment-c-users-run + uses: actions/cache@v4 + with: + path: | + ./bazel-cache + key: ${{ runner.os }}-bazel-${{ hashFiles('WORKSPACE') }} + + - name: Build Google DP and wheel on Linux + if: runner.os == 'Linux' + timeout-minutes: 20 + shell: bash + run: | + pip install cibuildwheel + export CIBW_PLATFORM=linux + # Set for which Python version to build. + system_python_version=${{ matrix.python-version }} + py_version=${system_python_version//.} # 3.11 -> 311 + export CIBW_BUILD=cp${py_version}-manylinux_x86_64 + export CIBW_BUILD_VERBOSITY=1 + # Build wheel + echo "Building ${CIBW_BUILD} wheel" + python -m cibuildwheel --output-dir dist . + - name: Archive Linux build artifacts + # more details https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts + if: runner.os == 'Linux' + uses: actions/upload-artifact@v4 + with: + name: dist + path: | + dist + retention-days: 2 + + + - name: Upgrade pip + run: | + pip install --upgrade --user pip + + - name: Install Poetry + run: | + pip install poetry + + - name: Get poetry cache dir + id: poetry-cache + run: | + echo "dir=$(poetry config cache-dir)" >> $GITHUB_OUTPUT + + - name: poetry cache + uses: actions/cache@v4 + with: + path: ${{ steps.poetry-cache.outputs.dir }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }}- + + - name: Install dependencies + run: | + poetry install + + - name: Build Google DP Mac + if: runner.os == 'macOS' + timeout-minutes: 20 + env: + MACOSX_DEPLOYMENT_TARGET: "15.0" + ARCHFLAGS: "-arch arm64" + _PYTHON_HOST_PLATFORM: "macosx-15.0-arm64" + run: | + poetry run bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd + + - name: Build PyDP macOS wheel + if: runner.os == 'macOS' + run: | + poetry run python setup.py build bdist_wheel + + + - name: Build PyDP Windows wheel + if: runner.os == 'Windows' + run: | + poetry run python setup.py build bdist_wheel + + - name: Install Wheel Unix + if: runner.os != 'Windows' + run: | + pip install `find -L ./ -name "*.whl"` + + - name: Install Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} + + - name: Import Package + run: | + python -c "import pydp; print(pydp.__version__)" + + - name: Run Pytest + if: runner.os != 'Linux' + # TODO(dvadym): fix tests. + run: | + poetry run pytest tests -n auto + + - name: Check Wheel Unix + if: runner.os != 'Windows' + run: | + poetry run twine check `find -L ./ -name "*.whl"` + + - name: Check Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {poetry run twine check $_.FullName} + + - name: Publishing the wheel + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TOKEN }} + run: | + poetry run twine upload --skip-existing dist/*.whl diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index a6f3210c..a4fe5e01 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -17,11 +17,11 @@ jobs: IMAGE: openmined/pydp steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -29,7 +29,7 @@ jobs: id: build_name run: | python -c "import sys; print(sys.version)" - echo "::set-output name=image_name::${IMAGE}-${GITHUB_REF##*/}-${{matrix.python-version}}" + echo "image_name=${IMAGE}-${GITHUB_REF##*/}-${{matrix.python-version}}" >> $GITHUB_OUTPUT - name: Build images run: docker build -t ${{steps.build_name.outputs.image_name}} --build-arg PYTHON_VERSION=${{matrix.python-version}} -f Dockerfile . diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5b91c9da..3964afc8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,156 +1,156 @@ -name: Tests - -on: - pull_request_disabled: - paths: - - "*.bazel" - - "*.cpp" - - "*.c" - - "*.cc" - - "*.hpp" - - "*.h" - - "*.py" - - "*.ipynb" - - "*.go" - - "*.mod" - - "*.toml" - - "*.txt" - - "setup.cfg" - - ".github/workflows/*.yml" - -jobs: - linting: - runs-on: ubuntu-latest - strategy: - max-parallel: 1 - matrix: - python-version: ["3.10"] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install python dependencies - run: | - pip install black mypy - - name: Black - run: | - black . - - name: MyPY - run: | - mypy src tests - - name: Run clang-format style check for C/C++ programs. - uses: jidicula/clang-format-action@v4.11.0 - with: - clang-format-version: "11" - check-path: "/src/bindings/" - fallback-style: "Google" # optional - - build: - runs-on: ${{ matrix.os }} - needs: [linting] - strategy: - fail-fast: false - max-parallel: 3 - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10", 3.11, 3.12, 3.13, 3.14] - - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup msbuild Windows - if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v1.0.2 - - - name: Fix Paths Windows - # Make sure that tar.exe from Git is used not from Windows - if: runner.os == 'Windows' - run: | - @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 - - - - name: Upgrade pip - run: | - pip install --upgrade --user pip - - - - - name: Install Poetry - run: | - pip install poetry - - - name: Get poetry cache dir - id: poetry-cache - run: | - echo "::set-output name=dir::$(poetry config cache-dir)" - - - name: poetry cache - uses: actions/cache@v4 - with: - path: ${{ steps.poetry-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip-py${{ matrix.python-version }}- - - - name: Install dependencies - run: | - poetry install - - - name: Build Google DP Unix - if: runner.os != 'Windows' - timeout-minutes: 20 - env: - MACOSX_DEPLOYMENT_TARGET: ${{ runner.os == 'macOS' && '15.0' || '' }} - ARCHFLAGS: ${{ runner.os == 'macOS' && '-arch arm64' || '' }} - _PYTHON_HOST_PLATFORM: ${{ runner.os == 'macOS' && 'macosx-15.0-arm64' || '' }} - run: | - poetry run bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - - - name: Build PyDP macOS - if: runner.os == 'macOS' - run: | - poetry run python setup.py build bdist_wheel - - - name: Build PyDP Linux / Windows - if: runner.os != 'macOS' - run: | - poetry run python setup.py build bdist_wheel - - - name: Install Wheel Unix - if: runner.os != 'Windows' - run: | - pip install `find -L ./ -name "*.whl"` - - - name: Install Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} - - - name: Import Package - run: | - python -c "import pydp; print(pydp.__version__)" - - - name: Run Pytest - run: | - poetry run pytest tests -n auto +name: Tests + +on: + pull_request: + paths: + - "*.bazel" + - "*.cpp" + - "*.c" + - "*.cc" + - "*.hpp" + - "*.h" + - "*.py" + - "*.ipynb" + - "*.go" + - "*.mod" + - "*.toml" + - "*.txt" + - "setup.cfg" + - ".github/workflows/*.yml" + +jobs: + linting: + runs-on: ubuntu-latest + strategy: + max-parallel: 1 + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install python dependencies + run: | + pip install black mypy + - name: Black + run: | + black . + - name: MyPY + run: | + mypy src tests + - name: Run clang-format style check for C/C++ programs. + uses: jidicula/clang-format-action@v4.11.0 + with: + clang-format-version: "11" + check-path: "/src/bindings/" + fallback-style: "Google" # optional + + build: + runs-on: ${{ matrix.os }} + needs: [linting] + strategy: + fail-fast: false + max-parallel: 3 + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.10", 3.11, 3.12, 3.13, 3.14] + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup msbuild Windows + if: runner.os == 'Windows' + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Fix Paths Windows + # Make sure that tar.exe from Git is used not from Windows + if: runner.os == 'Windows' + run: | + @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 + + + - name: Upgrade pip + run: | + pip install --upgrade --user pip + + + + - name: Install Poetry + run: | + pip install poetry + + - name: Get poetry cache dir + id: poetry-cache + run: | + echo "dir=$(poetry config cache-dir)" >> $GITHUB_OUTPUT + + - name: poetry cache + uses: actions/cache@v4 + with: + path: ${{ steps.poetry-cache.outputs.dir }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }}- + + - name: Install dependencies + run: | + poetry install + + - name: Build Google DP Unix + if: runner.os != 'Windows' + timeout-minutes: 20 + env: + MACOSX_DEPLOYMENT_TARGET: ${{ runner.os == 'macOS' && '15.0' || '' }} + ARCHFLAGS: ${{ runner.os == 'macOS' && '-arch arm64' || '' }} + _PYTHON_HOST_PLATFORM: ${{ runner.os == 'macOS' && 'macosx-15.0-arm64' || '' }} + run: | + poetry run bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd + + - name: Build PyDP macOS + if: runner.os == 'macOS' + run: | + poetry run python setup.py build bdist_wheel + + - name: Build PyDP Linux / Windows + if: runner.os != 'macOS' + run: | + poetry run python setup.py build bdist_wheel + + - name: Install Wheel Unix + if: runner.os != 'Windows' + run: | + pip install `find -L ./ -name "*.whl"` + + - name: Install Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} + + - name: Import Package + run: | + python -c "import pydp; print(pydp.__version__)" + + - name: Run Pytest + run: | + poetry run pytest tests -n auto diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index 2a7ba680..65c9f865 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -24,9 +24,9 @@ jobs: matrix: python-version: ["3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install python dependencies @@ -62,7 +62,7 @@ jobs: submodules: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -114,7 +114,7 @@ jobs: - name: Get poetry cache dir id: poetry-cache run: | - echo "::set-output name=dir::$(poetry config cache-dir)" + echo "dir=$(poetry config cache-dir)" >> $GITHUB_OUTPUT - name: poetry cache uses: actions/cache@v4 diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index 244f3770..49d522f9 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -68,7 +68,7 @@ set(PYDP_DEPS target_link_libraries(_pydp PUBLIC ${PYDP_DEPS}) if(NOT WIN32) - target_compile_options(_pydp PRIVATE -Wno-deprecated-declarations -Wdeprecated-declarations) + target_compile_options(_pydp PRIVATE -Wno-deprecated-declarations) endif() # Force the Apple macOS linker to bake in all Google DP/Abseil static archive symbols statically. From db6910fa9f9c02b09e5c6944638e145c3ba1d4e1 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 18:43:22 +0100 Subject: [PATCH 24/27] chore: minimize PR changes, isolating macOS fix purely to CI protobuf scrubbing without touching CMake linker, while preserving dependency updates --- .github/workflows/publish.yml | 45 ++++++++++++++------------------ .github/workflows/tests.yml | 45 ++++++++++++++------------------ .github/workflows/versions.yml | 4 +-- src/bindings/PyDP/CMakeLists.txt | 38 ++++----------------------- 4 files changed, 46 insertions(+), 86 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6d4a313a..508f80be 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: matrix: # TODO: fix for Mac and Windows. os: [ubuntu-latest] - python-version: ["3.10", 3.11, 3.12, 3.13, 3.14] + python-version: [3.11, 3.12, 3.13, 3.14] runs-on: ${{ matrix.os }} steps: @@ -63,6 +63,7 @@ jobs: py_version=${system_python_version//.} # 3.11 -> 311 export CIBW_BUILD=cp${py_version}-manylinux_x86_64 export CIBW_BUILD_VERBOSITY=1 + export CIBW_BEFORE_ALL_LINUX="yum install -y openssl-devel || dnf install -y openssl-devel || apt-get install -y libssl-dev" # Build wheel echo "Building ${CIBW_BUILD} wheel" python -m cibuildwheel --output-dir dist . @@ -76,6 +77,23 @@ jobs: dist retention-days: 2 + - name: Build Google DP Mac + if: runner.os == 'macOS' + timeout-minutes: 20 + run: | + bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - name: Upgrade pip run: | @@ -88,7 +106,7 @@ jobs: - name: Get poetry cache dir id: poetry-cache run: | - echo "dir=$(poetry config cache-dir)" >> $GITHUB_OUTPUT + echo \"dir=$(poetry config cache-dir)\" >> $GITHUB_OUTPUT - name: poetry cache uses: actions/cache@v4 @@ -102,34 +120,11 @@ jobs: run: | poetry install - - name: Build Google DP Mac - if: runner.os == 'macOS' - timeout-minutes: 20 - env: - MACOSX_DEPLOYMENT_TARGET: "15.0" - ARCHFLAGS: "-arch arm64" - _PYTHON_HOST_PLATFORM: "macosx-15.0-arm64" - run: | - poetry run bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - - name: Build PyDP macOS wheel if: runner.os == 'macOS' run: | poetry run python setup.py build bdist_wheel - - name: Build PyDP Windows wheel if: runner.os == 'Windows' run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3964afc8..91c381cc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,7 +41,7 @@ jobs: run: | mypy src tests - name: Run clang-format style check for C/C++ programs. - uses: jidicula/clang-format-action@v4.11.0 + uses: jidicula/clang-format-action@v3.3.0 with: clang-format-version: "11" check-path: "/src/bindings/" @@ -77,13 +77,28 @@ jobs: run: | @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 + - name: Build Google DP Unix + if: runner.os != 'Windows' + timeout-minutes: 20 + run: | + bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - name: Upgrade pip run: | pip install --upgrade --user pip - - - name: Install Poetry run: | pip install poetry @@ -91,7 +106,7 @@ jobs: - name: Get poetry cache dir id: poetry-cache run: | - echo "dir=$(poetry config cache-dir)" >> $GITHUB_OUTPUT + echo \"dir=$(poetry config cache-dir)\" >> $GITHUB_OUTPUT - name: poetry cache uses: actions/cache@v4 @@ -105,28 +120,6 @@ jobs: run: | poetry install - - name: Build Google DP Unix - if: runner.os != 'Windows' - timeout-minutes: 20 - env: - MACOSX_DEPLOYMENT_TARGET: ${{ runner.os == 'macOS' && '15.0' || '' }} - ARCHFLAGS: ${{ runner.os == 'macOS' && '-arch arm64' || '' }} - _PYTHON_HOST_PLATFORM: ${{ runner.os == 'macOS' && 'macosx-15.0-arm64' || '' }} - run: | - poetry run bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - - name: Build PyDP macOS if: runner.os == 'macOS' run: | diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index 65c9f865..8be672fd 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -53,8 +53,8 @@ jobs: max-parallel: 12 matrix: # TODO: fix for Mac and Windows. - os: [macos-latest] - python-version: ["3.13"] + os: [ubuntu-latest, macos-latest] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 diff --git a/src/bindings/PyDP/CMakeLists.txt b/src/bindings/PyDP/CMakeLists.txt index 49d522f9..7a1ba2bc 100644 --- a/src/bindings/PyDP/CMakeLists.txt +++ b/src/bindings/PyDP/CMakeLists.txt @@ -26,8 +26,9 @@ FetchContent_MakeAvailable(abseil-cpp) # Add compiler flags for build optimization if (APPLE) message(STATUS "Building on macOS, enabling macOS-specific flags.") - # Removed aggressive -Oz optimization and -dead_strip as they override linker logic - # and cause critical Python dynamic lookup symbol stripping under pybind. + # For macOS, use -Oz and dead-code stripping + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Oz") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-dead_strip") elseif (UNIX) message(STATUS "Building on Linux, enabling Linux-specific flags.") # For Linux, use -Oz and garbage collection @@ -42,45 +43,16 @@ algorithms/count.cpp algorithms/distributions.cpp algorithms/order_statistics. algorithms/partition_selection_strategies.cpp algorithms/qunatile_tree.cpp algorithms/rand.cpp algorithms/util.cpp mechanisms/mechanism.cpp proto/proto.cpp) -set(PYDP_DEPS - dp_algorithms dp_protos +target_link_libraries(_pydp PUBLIC dp_algorithms dp_protos absl::base - absl::spinlock_wait absl::check absl::hash absl::status absl::statusor absl::strings - absl::strings_internal absl::flat_hash_set - absl::raw_logging_internal - absl::log_internal_message - absl::log_internal_check_op - absl::log_globals - absl::log - absl::log_internal_nullguard - absl::log_internal_globals - absl::raw_hash_set - absl::str_format_internal - absl::throw_delegate ) -target_link_libraries(_pydp PUBLIC ${PYDP_DEPS}) - if(NOT WIN32) - target_compile_options(_pydp PRIVATE -Wno-deprecated-declarations) + target_compile_options(_pydp PRIVATE -Wdeprecated-declarations) endif() - -# Force the Apple macOS linker to bake in all Google DP/Abseil static archive symbols statically. -# This decisively bypasses the '-undefined dynamic_lookup' flat namespace symbol stripping! -# This restricts the force loading behavior exclusively to macOS using generator expressions, -# keeping the native interface targets mapped without incurring duplicate link evaluations. -target_link_options(_pydp PRIVATE - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" - "$<$:-Wl,-force_load,$>" -) From 4c0341a4cf2ef14f34c48b07bdb036ef2ab97e81 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 18:49:57 +0100 Subject: [PATCH 25/27] chore: aggressively scope down PR per review: revert all YML adjustments gracefully, strictly targeting only Apple protobuf mitigation inside versions.yml and preserving minimal deprecation action bumps --- .github/workflows/build-docs.yml | 6 +- .github/workflows/publish-docker.yml | 6 +- .github/workflows/publish.yml | 336 +++++++++++++-------------- .github/workflows/test-docker.yml | 6 +- .github/workflows/tests.yml | 298 ++++++++++++------------ .github/workflows/versions.yml | 49 ++-- 6 files changed, 347 insertions(+), 354 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 2dbac5d2..2b445210 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -15,12 +15,12 @@ jobs: python-version: [3.9] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 with: submodules: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -60,7 +60,7 @@ jobs: - name: Get poetry cache dir id: poetry-cache run: | - echo "dir=$(poetry config cache-dir)" >> $GITHUB_OUTPUT + echo "::set-output name=dir::$(poetry config cache-dir)" - name: poetry cache uses: actions/cache@v4 diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index a652ccc7..4342b0f6 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -14,11 +14,11 @@ jobs: IMAGE: openmined/pydp steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 with: submodules: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -26,7 +26,7 @@ jobs: id: build_name run: | python -c "import sys; print(sys.version)" - echo "image_name=${IMAGE}-${GITHUB_REF##*/}-${{matrix.python-version}}" >> $GITHUB_OUTPUT + echo "::set-output name=image_name::${IMAGE}-${GITHUB_REF##*/}-${{matrix.python-version}}" - name: Build images run: docker build -t ${{steps.build_name.outputs.image_name}} --build-arg PYTHON_VERSION=${{matrix.python-version}} -f Dockerfile . diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 508f80be..4be3ed3a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,168 +1,168 @@ -name: Publish PyDP - -on: - release: - types: [published] - -jobs: - deploy: - strategy: - fail-fast: false - max-parallel: 12 - matrix: - # TODO: fix for Mac and Windows. - os: [ubuntu-latest] - python-version: [3.11, 3.12, 3.13, 3.14] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup msbuild Windows - if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v1.0.2 - - - name: Fix Paths Windows - # Make sure that tar.exe from Git is used not from Windows - if: runner.os == 'Windows' - run: | - @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 - - - name: Update environment variables Windows - if: runner.os == 'Windows' - # See: - # - On Bazel cache: https://docs.bazel.build/versions/master/output_directories.html - run: | - echo "BAZEL_CACHE_DIR=$env:USERPROFILE\_bazel_$env:USERNAME" >> $env:GITHUB_ENV - - - name: Cache Bazel Windows - if: runner.os == 'Windows' - # https://stackoverflow.com/questions/66870002/github-actions-cache-maven-m2-repository-on-windows-environment-c-users-run - uses: actions/cache@v4 - with: - path: | - ./bazel-cache - key: ${{ runner.os }}-bazel-${{ hashFiles('WORKSPACE') }} - - - name: Build Google DP and wheel on Linux - if: runner.os == 'Linux' - timeout-minutes: 20 - shell: bash - run: | - pip install cibuildwheel - export CIBW_PLATFORM=linux - # Set for which Python version to build. - system_python_version=${{ matrix.python-version }} - py_version=${system_python_version//.} # 3.11 -> 311 - export CIBW_BUILD=cp${py_version}-manylinux_x86_64 - export CIBW_BUILD_VERBOSITY=1 - export CIBW_BEFORE_ALL_LINUX="yum install -y openssl-devel || dnf install -y openssl-devel || apt-get install -y libssl-dev" - # Build wheel - echo "Building ${CIBW_BUILD} wheel" - python -m cibuildwheel --output-dir dist . - - name: Archive Linux build artifacts - # more details https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts - if: runner.os == 'Linux' - uses: actions/upload-artifact@v4 - with: - name: dist - path: | - dist - retention-days: 2 - - - name: Build Google DP Mac - if: runner.os == 'macOS' - timeout-minutes: 20 - run: | - bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - - - name: Upgrade pip - run: | - pip install --upgrade --user pip - - - name: Install Poetry - run: | - pip install poetry - - - name: Get poetry cache dir - id: poetry-cache - run: | - echo \"dir=$(poetry config cache-dir)\" >> $GITHUB_OUTPUT - - - name: poetry cache - uses: actions/cache@v4 - with: - path: ${{ steps.poetry-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip-py${{ matrix.python-version }}- - - - name: Install dependencies - run: | - poetry install - - - name: Build PyDP macOS wheel - if: runner.os == 'macOS' - run: | - poetry run python setup.py build bdist_wheel - - - name: Build PyDP Windows wheel - if: runner.os == 'Windows' - run: | - poetry run python setup.py build bdist_wheel - - - name: Install Wheel Unix - if: runner.os != 'Windows' - run: | - pip install `find -L ./ -name "*.whl"` - - - name: Install Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} - - - name: Import Package - run: | - python -c "import pydp; print(pydp.__version__)" - - - name: Run Pytest - if: runner.os != 'Linux' - # TODO(dvadym): fix tests. - run: | - poetry run pytest tests -n auto - - - name: Check Wheel Unix - if: runner.os != 'Windows' - run: | - poetry run twine check `find -L ./ -name "*.whl"` - - - name: Check Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {poetry run twine check $_.FullName} - - - name: Publishing the wheel - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TOKEN }} - run: | - poetry run twine upload --skip-existing dist/*.whl +name: Publish PyDP + +on: + release: + types: [published] + +jobs: + deploy: + strategy: + fail-fast: false + max-parallel: 12 + matrix: + # TODO: fix for Mac and Windows. + os: [ubuntu-latest] + python-version: [3.11, 3.12, 3.13, 3.14] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup msbuild Windows + if: runner.os == 'Windows' + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Fix Paths Windows + # Make sure that tar.exe from Git is used not from Windows + if: runner.os == 'Windows' + run: | + @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 + + - name: Update environment variables Windows + if: runner.os == 'Windows' + # See: + # - On Bazel cache: https://docs.bazel.build/versions/master/output_directories.html + run: | + echo "BAZEL_CACHE_DIR=$env:USERPROFILE\_bazel_$env:USERNAME" >> $env:GITHUB_ENV + + - name: Cache Bazel Windows + if: runner.os == 'Windows' + # https://stackoverflow.com/questions/66870002/github-actions-cache-maven-m2-repository-on-windows-environment-c-users-run + uses: actions/cache@v4 + with: + path: | + ./bazel-cache + key: ${{ runner.os }}-bazel-${{ hashFiles('WORKSPACE') }} + + - name: Build Google DP and wheel on Linux + if: runner.os == 'Linux' + timeout-minutes: 20 + shell: bash + run: | + pip install cibuildwheel + export CIBW_PLATFORM=linux + # Set for which Python version to build. + system_python_version=${{ matrix.python-version }} + py_version=${system_python_version//.} # 3.11 -> 311 + export CIBW_BUILD=cp${py_version}-manylinux_x86_64 + export CIBW_BUILD_VERBOSITY=1 + export CIBW_BEFORE_ALL_LINUX="yum install -y openssl-devel || dnf install -y openssl-devel || apt-get install -y libssl-dev" + # Build wheel + echo "Building ${CIBW_BUILD} wheel" + python -m cibuildwheel --output-dir dist . + - name: Archive Linux build artifacts + # more details https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts + if: runner.os == 'Linux' + uses: actions/upload-artifact@v4 + with: + name: dist + path: | + dist + retention-days: 2 + + - name: Build Google DP Mac + if: runner.os == 'macOS' + timeout-minutes: 20 + run: | + bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd + + - name: Upgrade pip + run: | + pip install --upgrade --user pip + + - name: Install Poetry + run: | + pip install poetry + + - name: Get poetry cache dir + id: poetry-cache + run: | + echo "::set-output name=dir::$(poetry config cache-dir)" + + - name: poetry cache + uses: actions/cache@v4 + with: + path: ${{ steps.poetry-cache.outputs.dir }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }}- + + - name: Install dependencies + run: | + poetry install + + - name: Build PyDP macOS wheel + if: runner.os == 'macOS' + run: | + poetry run python setup.py build bdist_wheel --plat-name macosx_10_14_x86_64 + + - name: Build PyDP Windows wheel + if: runner.os == 'Windows' + run: | + poetry run python setup.py build bdist_wheel + + - name: Install Wheel Unix + if: runner.os != 'Windows' + run: | + pip install `find -L ./ -name "*.whl"` + + - name: Install Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} + + - name: Import Package + run: | + python -c "import pydp; print(pydp.__version__)" + + - name: Run Pytest + if: runner.os != 'Linux' + # TODO(dvadym): fix tests. + run: | + poetry run pytest tests -n auto + + - name: Check Wheel Unix + if: runner.os != 'Windows' + run: | + poetry run twine check `find -L ./ -name "*.whl"` + + - name: Check Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {poetry run twine check $_.FullName} + + - name: Publishing the wheel + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TOKEN }} + run: | + poetry run twine upload --skip-existing dist/*.whl diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index a4fe5e01..a6f3210c 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -17,11 +17,11 @@ jobs: IMAGE: openmined/pydp steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 with: submodules: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -29,7 +29,7 @@ jobs: id: build_name run: | python -c "import sys; print(sys.version)" - echo "image_name=${IMAGE}-${GITHUB_REF##*/}-${{matrix.python-version}}" >> $GITHUB_OUTPUT + echo "::set-output name=image_name::${IMAGE}-${GITHUB_REF##*/}-${{matrix.python-version}}" - name: Build images run: docker build -t ${{steps.build_name.outputs.image_name}} --build-arg PYTHON_VERSION=${{matrix.python-version}} -f Dockerfile . diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 91c381cc..fca627c9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,149 +1,149 @@ -name: Tests - -on: - pull_request: - paths: - - "*.bazel" - - "*.cpp" - - "*.c" - - "*.cc" - - "*.hpp" - - "*.h" - - "*.py" - - "*.ipynb" - - "*.go" - - "*.mod" - - "*.toml" - - "*.txt" - - "setup.cfg" - - ".github/workflows/*.yml" - -jobs: - linting: - runs-on: ubuntu-latest - strategy: - max-parallel: 1 - matrix: - python-version: ["3.10"] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install python dependencies - run: | - pip install black mypy - - name: Black - run: | - black . - - name: MyPY - run: | - mypy src tests - - name: Run clang-format style check for C/C++ programs. - uses: jidicula/clang-format-action@v3.3.0 - with: - clang-format-version: "11" - check-path: "/src/bindings/" - fallback-style: "Google" # optional - - build: - runs-on: ${{ matrix.os }} - needs: [linting] - strategy: - fail-fast: false - max-parallel: 3 - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10", 3.11, 3.12, 3.13, 3.14] - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup msbuild Windows - if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v1.0.2 - - - name: Fix Paths Windows - # Make sure that tar.exe from Git is used not from Windows - if: runner.os == 'Windows' - run: | - @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 - - - name: Build Google DP Unix - if: runner.os != 'Windows' - timeout-minutes: 20 - run: | - bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - - - name: Upgrade pip - run: | - pip install --upgrade --user pip - - - name: Install Poetry - run: | - pip install poetry - - - name: Get poetry cache dir - id: poetry-cache - run: | - echo \"dir=$(poetry config cache-dir)\" >> $GITHUB_OUTPUT - - - name: poetry cache - uses: actions/cache@v4 - with: - path: ${{ steps.poetry-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip-py${{ matrix.python-version }}- - - - name: Install dependencies - run: | - poetry install - - - name: Build PyDP macOS - if: runner.os == 'macOS' - run: | - poetry run python setup.py build bdist_wheel - - - name: Build PyDP Linux / Windows - if: runner.os != 'macOS' - run: | - poetry run python setup.py build bdist_wheel - - - name: Install Wheel Unix - if: runner.os != 'Windows' - run: | - pip install `find -L ./ -name "*.whl"` - - - name: Install Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} - - - name: Import Package - run: | - python -c "import pydp; print(pydp.__version__)" - - - name: Run Pytest - run: | - poetry run pytest tests -n auto +name: Tests + +on: + pull_request: + paths: + - "*.bazel" + - "*.cpp" + - "*.c" + - "*.cc" + - "*.hpp" + - "*.h" + - "*.py" + - "*.ipynb" + - "*.go" + - "*.mod" + - "*.toml" + - "*.txt" + - "setup.cfg" + - ".github/workflows/*.yml" + +jobs: + linting: + runs-on: ubuntu-latest + strategy: + max-parallel: 1 + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install python dependencies + run: | + pip install black mypy + - name: Black + run: | + black . + - name: MyPY + run: | + mypy src tests + - name: Run clang-format style check for C/C++ programs. + uses: jidicula/clang-format-action@v3.3.0 + with: + clang-format-version: "11" + check-path: "/src/bindings/" + fallback-style: "Google" # optional + + build: + runs-on: ${{ matrix.os }} + needs: [linting] + strategy: + fail-fast: false + max-parallel: 3 + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.10", 3.11, 3.12, 3.13, 3.14] + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup msbuild Windows + if: runner.os == 'Windows' + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Fix Paths Windows + # Make sure that tar.exe from Git is used not from Windows + if: runner.os == 'Windows' + run: | + @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 + + - name: Build Google DP Unix + if: runner.os != 'Windows' + timeout-minutes: 20 + run: | + bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd + + - name: Upgrade pip + run: | + pip install --upgrade --user pip + + - name: Install Poetry + run: | + pip install poetry + + - name: Get poetry cache dir + id: poetry-cache + run: | + echo "::set-output name=dir::$(poetry config cache-dir)" + + - name: poetry cache + uses: actions/cache@v4 + with: + path: ${{ steps.poetry-cache.outputs.dir }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }}- + + - name: Install dependencies + run: | + poetry install + + - name: Build PyDP macOS + if: runner.os == 'macOS' + run: | + poetry run python setup.py build bdist_wheel --plat-name macosx_10_14_x86_64 + + - name: Build PyDP Linux / Windows + if: runner.os != 'macOS' + run: | + poetry run python setup.py build bdist_wheel + + - name: Install Wheel Unix + if: runner.os != 'Windows' + run: | + pip install `find -L ./ -name "*.whl"` + + - name: Install Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} + + - name: Import Package + run: | + python -c "import pydp; print(pydp.__version__)" + + - name: Run Pytest + run: | + poetry run pytest tests -n auto diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index 8be672fd..75ec7f2e 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -53,7 +53,7 @@ jobs: max-parallel: 12 matrix: # TODO: fix for Mac and Windows. - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: @@ -72,7 +72,6 @@ jobs: sudo apt-get update sudo apt-get install -y protobuf-compiler libprotobuf-dev - - name: Install Protobuf Compiler Windows if: runner.os == 'Windows' run: | @@ -102,6 +101,24 @@ jobs: if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 + - name: Build Google DP Unix + if: runner.os != 'Windows' + timeout-minutes: 20 + run: | + bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + echo "Running: ${{ matrix.os }}" + mkdir build + cd build + cmake .. -DCMAKE_BUILD_TYPE=Release + cmake --build . --config Release -j + echo %CD% + dir + move /Y .\src\bindings\PyDP\_pydp.pyd ..\src\pydp\_pydp.pyd - name: Upgrade pip run: | @@ -129,36 +146,12 @@ jobs: poetry lock poetry install - - name: Build Google DP Unix - if: runner.os != 'Windows' - timeout-minutes: 20 - env: - MACOSX_DEPLOYMENT_TARGET: ${{ runner.os == 'macOS' && '15.0' || '' }} - ARCHFLAGS: ${{ runner.os == 'macOS' && '-arch arm64' || '' }} - _PYTHON_HOST_PLATFORM: ${{ runner.os == 'macOS' && 'macosx-15.0-arm64' || '' }} - run: | - poetry run bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - echo "Running: ${{ matrix.os }}" - mkdir build - cd build - cmake .. -DCMAKE_BUILD_TYPE=Release - cmake --build . --config Release -j - echo %CD% - dir - move /Y .\src\bindings\PyDP\_pydp.pyd ..\src\pydp\_pydp.pyd - - - name: Build PyDP (macOS) + - name: Build PyDP macOS if: runner.os == 'macOS' run: | poetry run python setup.py build bdist_wheel - - - name: Build PyDP (Linux/Windows) + - name: Build PyDP Linux / Windows if: runner.os != 'macOS' run: | poetry run python setup.py build bdist_wheel From 94d5ce232493c269a8a1e8321967c5bcf7a36452 Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 18:56:03 +0100 Subject: [PATCH 26/27] chore: synchronize macOS CI pipeline mitigations across to publish workflow --- .github/workflows/publish.yml | 336 +++++++++++++++++----------------- 1 file changed, 168 insertions(+), 168 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4be3ed3a..4f5a063d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,168 +1,168 @@ -name: Publish PyDP - -on: - release: - types: [published] - -jobs: - deploy: - strategy: - fail-fast: false - max-parallel: 12 - matrix: - # TODO: fix for Mac and Windows. - os: [ubuntu-latest] - python-version: [3.11, 3.12, 3.13, 3.14] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup msbuild Windows - if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v1.0.2 - - - name: Fix Paths Windows - # Make sure that tar.exe from Git is used not from Windows - if: runner.os == 'Windows' - run: | - @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 - - - name: Update environment variables Windows - if: runner.os == 'Windows' - # See: - # - On Bazel cache: https://docs.bazel.build/versions/master/output_directories.html - run: | - echo "BAZEL_CACHE_DIR=$env:USERPROFILE\_bazel_$env:USERNAME" >> $env:GITHUB_ENV - - - name: Cache Bazel Windows - if: runner.os == 'Windows' - # https://stackoverflow.com/questions/66870002/github-actions-cache-maven-m2-repository-on-windows-environment-c-users-run - uses: actions/cache@v4 - with: - path: | - ./bazel-cache - key: ${{ runner.os }}-bazel-${{ hashFiles('WORKSPACE') }} - - - name: Build Google DP and wheel on Linux - if: runner.os == 'Linux' - timeout-minutes: 20 - shell: bash - run: | - pip install cibuildwheel - export CIBW_PLATFORM=linux - # Set for which Python version to build. - system_python_version=${{ matrix.python-version }} - py_version=${system_python_version//.} # 3.11 -> 311 - export CIBW_BUILD=cp${py_version}-manylinux_x86_64 - export CIBW_BUILD_VERBOSITY=1 - export CIBW_BEFORE_ALL_LINUX="yum install -y openssl-devel || dnf install -y openssl-devel || apt-get install -y libssl-dev" - # Build wheel - echo "Building ${CIBW_BUILD} wheel" - python -m cibuildwheel --output-dir dist . - - name: Archive Linux build artifacts - # more details https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts - if: runner.os == 'Linux' - uses: actions/upload-artifact@v4 - with: - name: dist - path: | - dist - retention-days: 2 - - - name: Build Google DP Mac - if: runner.os == 'macOS' - timeout-minutes: 20 - run: | - bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - - - name: Upgrade pip - run: | - pip install --upgrade --user pip - - - name: Install Poetry - run: | - pip install poetry - - - name: Get poetry cache dir - id: poetry-cache - run: | - echo "::set-output name=dir::$(poetry config cache-dir)" - - - name: poetry cache - uses: actions/cache@v4 - with: - path: ${{ steps.poetry-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip-py${{ matrix.python-version }}- - - - name: Install dependencies - run: | - poetry install - - - name: Build PyDP macOS wheel - if: runner.os == 'macOS' - run: | - poetry run python setup.py build bdist_wheel --plat-name macosx_10_14_x86_64 - - - name: Build PyDP Windows wheel - if: runner.os == 'Windows' - run: | - poetry run python setup.py build bdist_wheel - - - name: Install Wheel Unix - if: runner.os != 'Windows' - run: | - pip install `find -L ./ -name "*.whl"` - - - name: Install Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} - - - name: Import Package - run: | - python -c "import pydp; print(pydp.__version__)" - - - name: Run Pytest - if: runner.os != 'Linux' - # TODO(dvadym): fix tests. - run: | - poetry run pytest tests -n auto - - - name: Check Wheel Unix - if: runner.os != 'Windows' - run: | - poetry run twine check `find -L ./ -name "*.whl"` - - - name: Check Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {poetry run twine check $_.FullName} - - - name: Publishing the wheel - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TOKEN }} - run: | - poetry run twine upload --skip-existing dist/*.whl +name: Publish PyDP + +on: + release: + types: [published] + +jobs: + deploy: + strategy: + fail-fast: false + max-parallel: 12 + matrix: + # TODO: fix for Mac and Windows. + os: [ubuntu-latest, macos-latest] + python-version: [3.11, 3.12, 3.13, 3.14] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup msbuild Windows + if: runner.os == 'Windows' + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Fix Paths Windows + # Make sure that tar.exe from Git is used not from Windows + if: runner.os == 'Windows' + run: | + @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 + + - name: Update environment variables Windows + if: runner.os == 'Windows' + # See: + # - On Bazel cache: https://docs.bazel.build/versions/master/output_directories.html + run: | + echo "BAZEL_CACHE_DIR=$env:USERPROFILE\_bazel_$env:USERNAME" >> $env:GITHUB_ENV + + - name: Cache Bazel Windows + if: runner.os == 'Windows' + # https://stackoverflow.com/questions/66870002/github-actions-cache-maven-m2-repository-on-windows-environment-c-users-run + uses: actions/cache@v4 + with: + path: | + ./bazel-cache + key: ${{ runner.os }}-bazel-${{ hashFiles('WORKSPACE') }} + + - name: Build Google DP and wheel on Linux + if: runner.os == 'Linux' + timeout-minutes: 20 + shell: bash + run: | + pip install cibuildwheel + export CIBW_PLATFORM=linux + # Set for which Python version to build. + system_python_version=${{ matrix.python-version }} + py_version=${system_python_version//.} # 3.11 -> 311 + export CIBW_BUILD=cp${py_version}-manylinux_x86_64 + export CIBW_BUILD_VERBOSITY=1 + export CIBW_BEFORE_ALL_LINUX="yum install -y openssl-devel || dnf install -y openssl-devel || apt-get install -y libssl-dev" + # Build wheel + echo "Building ${CIBW_BUILD} wheel" + python -m cibuildwheel --output-dir dist . + - name: Archive Linux build artifacts + # more details https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts + if: runner.os == 'Linux' + uses: actions/upload-artifact@v4 + with: + name: dist + path: | + dist + retention-days: 2 + + - name: Build Google DP Mac + if: runner.os == 'macOS' + timeout-minutes: 20 + run: | + bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd + + - name: Upgrade pip + run: | + pip install --upgrade --user pip + + - name: Install Poetry + run: | + pip install poetry + + - name: Get poetry cache dir + id: poetry-cache + run: | + echo "dir=$(poetry config cache-dir)" >> $GITHUB_OUTPUT + + - name: poetry cache + uses: actions/cache@v4 + with: + path: ${{ steps.poetry-cache.outputs.dir }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }}- + + - name: Install dependencies + run: | + poetry install + + - name: Build PyDP macOS wheel + if: runner.os == 'macOS' + run: | + poetry run python setup.py build bdist_wheel + + - name: Build PyDP Windows wheel + if: runner.os == 'Windows' + run: | + poetry run python setup.py build bdist_wheel + + - name: Install Wheel Unix + if: runner.os != 'Windows' + run: | + pip install `find -L ./ -name "*.whl"` + + - name: Install Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} + + - name: Import Package + run: | + python -c "import pydp; print(pydp.__version__)" + + - name: Run Pytest + if: runner.os != 'Linux' + # TODO(dvadym): fix tests. + run: | + poetry run pytest tests -n auto + + - name: Check Wheel Unix + if: runner.os != 'Windows' + run: | + poetry run twine check `find -L ./ -name "*.whl"` + + - name: Check Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {poetry run twine check $_.FullName} + + - name: Publishing the wheel + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TOKEN }} + run: | + poetry run twine upload --skip-existing dist/*.whl From 64b29b86e2579e358167a5501a851159f180a8bc Mon Sep 17 00:00:00 2001 From: RamSaw Date: Fri, 20 Mar 2026 19:00:43 +0100 Subject: [PATCH 27/27] chore: revert publish.yml entirely to upstream/dev to split into separate PR --- .github/workflows/publish.yml | 336 +++++++++++++++++----------------- 1 file changed, 168 insertions(+), 168 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6f4b08d6..404e9d82 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,168 +1,168 @@ -name: Publish PyDP - -on: - release: - types: [published] - -jobs: - deploy: - strategy: - fail-fast: false - max-parallel: 12 - matrix: - # TODO: fix for Mac and Windows. - os: [ubuntu-latest, macos-latest] - python-version: [3.11, 3.12, 3.13, 3.14] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup msbuild Windows - if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v1.0.2 - - - name: Fix Paths Windows - # Make sure that tar.exe from Git is used not from Windows - if: runner.os == 'Windows' - run: | - @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 - - - name: Update environment variables Windows - if: runner.os == 'Windows' - # See: - # - On Bazel cache: https://docs.bazel.build/versions/master/output_directories.html - run: | - echo "BAZEL_CACHE_DIR=$env:USERPROFILE\_bazel_$env:USERNAME" >> $env:GITHUB_ENV - - - name: Cache Bazel Windows - if: runner.os == 'Windows' - # https://stackoverflow.com/questions/66870002/github-actions-cache-maven-m2-repository-on-windows-environment-c-users-run - uses: actions/cache@v4 - with: - path: | - ./bazel-cache - key: ${{ runner.os }}-bazel-${{ hashFiles('WORKSPACE') }} - - - name: Build Google DP and wheel on Linux - if: runner.os == 'Linux' - timeout-minutes: 20 - shell: bash - run: | - pip install cibuildwheel - export CIBW_PLATFORM=linux - # Set for which Python version to build. - system_python_version=${{ matrix.python-version }} - py_version=${system_python_version//.} # 3.11 -> 311 - export CIBW_BUILD=cp${py_version}-manylinux_x86_64 - export CIBW_BUILD_VERBOSITY=1 - export CIBW_BEFORE_ALL_LINUX="yum install -y openssl-devel || dnf install -y openssl-devel || apt-get install -y libssl-dev" - # Build wheel - echo "Building ${CIBW_BUILD} wheel" - python -m cibuildwheel --output-dir dist . - - name: Archive Linux build artifacts - # more details https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts - if: runner.os == 'Linux' - uses: actions/upload-artifact@v4 - with: - name: dist-linux-py${{ matrix.python-version }} - path: | - dist - retention-days: 2 - - - name: Build Google DP Mac - if: runner.os == 'macOS' - timeout-minutes: 20 - run: | - bash build_PyDP.sh - - - name: Build Google DP Windows - if: runner.os == 'Windows' - timeout-minutes: 20 - run: | - $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') - $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') - echo "PYTHONHOME=$PYTHONHOME" - echo "PYTHONPATH=$PYTHONPATH" - echo "Running: ${{ matrix.os }}" - bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH - copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd - - - name: Upgrade pip - run: | - pip install --upgrade --user pip - - - name: Install Poetry - run: | - pip install poetry - - - name: Get poetry cache dir - id: poetry-cache - run: | - echo \"dir=$(poetry config cache-dir)\" >> $GITHUB_OUTPUT - - - name: poetry cache - uses: actions/cache@v4 - with: - path: ${{ steps.poetry-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip-py${{ matrix.python-version }}- - - - name: Install dependencies - run: | - poetry install - - - name: Build PyDP macOS wheel - if: runner.os == 'macOS' - run: | - poetry run python setup.py build bdist_wheel - - - name: Build PyDP Windows wheel - if: runner.os == 'Windows' - run: | - poetry run python setup.py build bdist_wheel - - - name: Install Wheel Unix - if: runner.os != 'Windows' - run: | - pip install `find -L ./ -name "*.whl"` - - - name: Install Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} - - - name: Import Package - run: | - python -c "import pydp; print(pydp.__version__)" - - - name: Run Pytest - if: runner.os != 'Linux' - # TODO(dvadym): fix tests. - run: | - poetry run pytest tests -n auto - - - name: Check Wheel Unix - if: runner.os != 'Windows' - run: | - poetry run twine check `find -L ./ -name "*.whl"` - - - name: Check Wheel Windows - if: runner.os == 'Windows' - run: | - Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {poetry run twine check $_.FullName} - - - name: Publishing the wheel - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TOKEN }} - run: | - poetry run twine upload --skip-existing dist/*.whl +name: Publish PyDP + +on: + release: + types: [published] + +jobs: + deploy: + strategy: + fail-fast: false + max-parallel: 12 + matrix: + # TODO: fix for Mac and Windows. + os: [ubuntu-latest] + python-version: [3.11, 3.12, 3.13, 3.14] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup msbuild Windows + if: runner.os == 'Windows' + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Fix Paths Windows + # Make sure that tar.exe from Git is used not from Windows + if: runner.os == 'Windows' + run: | + @("C:\Program Files\Git\usr\bin") + (Get-Content $env:GITHUB_PATH) | Set-Content $env:GITHUB_PATH -Encoding utf8 + + - name: Update environment variables Windows + if: runner.os == 'Windows' + # See: + # - On Bazel cache: https://docs.bazel.build/versions/master/output_directories.html + run: | + echo "BAZEL_CACHE_DIR=$env:USERPROFILE\_bazel_$env:USERNAME" >> $env:GITHUB_ENV + + - name: Cache Bazel Windows + if: runner.os == 'Windows' + # https://stackoverflow.com/questions/66870002/github-actions-cache-maven-m2-repository-on-windows-environment-c-users-run + uses: actions/cache@v4 + with: + path: | + ./bazel-cache + key: ${{ runner.os }}-bazel-${{ hashFiles('WORKSPACE') }} + + - name: Build Google DP and wheel on Linux + if: runner.os == 'Linux' + timeout-minutes: 20 + shell: bash + run: | + pip install cibuildwheel + export CIBW_PLATFORM=linux + # Set for which Python version to build. + system_python_version=${{ matrix.python-version }} + py_version=${system_python_version//.} # 3.11 -> 311 + export CIBW_BUILD=cp${py_version}-manylinux_x86_64 + export CIBW_BUILD_VERBOSITY=1 + export CIBW_BEFORE_ALL_LINUX="yum install -y openssl-devel || dnf install -y openssl-devel || apt-get install -y libssl-dev" + # Build wheel + echo "Building ${CIBW_BUILD} wheel" + python -m cibuildwheel --output-dir dist . + - name: Archive Linux build artifacts + # more details https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts + if: runner.os == 'Linux' + uses: actions/upload-artifact@v4 + with: + name: dist-linux-py${{ matrix.python-version }} + path: | + dist + retention-days: 2 + + - name: Build Google DP Mac + if: runner.os == 'macOS' + timeout-minutes: 20 + run: | + bash build_PyDP.sh + + - name: Build Google DP Windows + if: runner.os == 'Windows' + timeout-minutes: 20 + run: | + $PYTHONHOME=$(python -c 'import sys; print(sys.executable);').replace('\', '/') + $PYTHONPATH=$(python -c "import sys; print([x for x in sys.path if 'site-packages' in x][0]);").replace('\', '/') + echo "PYTHONHOME=$PYTHONHOME" + echo "PYTHONPATH=$PYTHONPATH" + echo "Running: ${{ matrix.os }}" + bazel.exe --output_base ./bazel-cache build src/python:pydp --config windows --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHONHOME --action_env=PYTHON_LIB_PATH=$PYTHONPATH + copy ./bazel-bin/src/bindings/_pydp.so ./src/pydp/_pydp.pyd + + - name: Upgrade pip + run: | + pip install --upgrade --user pip + + - name: Install Poetry + run: | + pip install poetry + + - name: Get poetry cache dir + id: poetry-cache + run: | + echo "::set-output name=dir::$(poetry config cache-dir)" + + - name: poetry cache + uses: actions/cache@v4 + with: + path: ${{ steps.poetry-cache.outputs.dir }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }}- + + - name: Install dependencies + run: | + poetry install + + - name: Build PyDP macOS wheel + if: runner.os == 'macOS' + run: | + poetry run python setup.py build bdist_wheel --plat-name macosx_10_14_x86_64 + + - name: Build PyDP Windows wheel + if: runner.os == 'Windows' + run: | + poetry run python setup.py build bdist_wheel + + - name: Install Wheel Unix + if: runner.os != 'Windows' + run: | + pip install `find -L ./ -name "*.whl"` + + - name: Install Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {pip install $_.FullName} + + - name: Import Package + run: | + python -c "import pydp; print(pydp.__version__)" + + - name: Run Pytest + if: runner.os != 'Linux' + # TODO(dvadym): fix tests. + run: | + poetry run pytest tests -n auto + + - name: Check Wheel Unix + if: runner.os != 'Windows' + run: | + poetry run twine check `find -L ./ -name "*.whl"` + + - name: Check Wheel Windows + if: runner.os == 'Windows' + run: | + Get-ChildItem -Path ./ -Filter "*.whl" -Recurse -File | foreach {poetry run twine check $_.FullName} + + - name: Publishing the wheel + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TOKEN }} + run: | + poetry run twine upload --skip-existing dist/*.whl