From c4fc4d5721e3b265c62cbcc97f121eb984ba9f73 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 13:37:46 +0000 Subject: [PATCH 1/5] Initial plan From d5920da3b16830c3e036ec073fb0cc4352c91ab4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 13:45:05 +0000 Subject: [PATCH 2/5] Add BUILD_PYOPENMS option to Dockerfile and Windows workflows Co-authored-by: t0mdavid-m <57191390+t0mdavid-m@users.noreply.github.com> --- ...ndows-executable-app-with-pyinstaller.yaml | 39 +++++++- .../build-windows-executable-app.yaml | 47 +++++++++- Dockerfile | 88 ++++++++++++------- 3 files changed, 137 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml b/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml index 2d3b6f14..2958b2cd 100644 --- a/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml +++ b/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml @@ -3,6 +3,8 @@ on: workflow_dispatch: env: OPENMS_VERSION: 3.2.0 + # Control whether to build pyOpenMS from source (true) or use pip (false). Default: false + BUILD_PYOPENMS: false # Define needed TOPP tools here TOPP_TOOLS: "FeatureFinderMetabo MetaboliteAdductDecharger SiriusExport" @@ -133,6 +135,7 @@ jobs: BUILD_TYPE: "Release" OPENMP: "Off" USE_STATIC_BOOST: "On" + PYOPENMS: "${{ env.BUILD_PYOPENMS == 'true' && 'ON' || 'OFF' }}" # BUILD_FLAGS: "-p:CL_MPCount=2" # For VS Generator and MSBuild BUILD_FLAGS: "-j2" # Ninja will otherwise use all cores (doesn't go well in GHA) CMAKE_CCACHE_EXE: "ccache" @@ -142,6 +145,28 @@ jobs: CCACHE_COMPRESSLEVEL: 12 CCACHE_MAXSIZE: 400M + - name: Build pyOpenMS + if: env.BUILD_PYOPENMS == 'true' + shell: bash + run: | + cd $GITHUB_WORKSPACE/OpenMS/bld/ + ninja pyopenms + + - name: Package pyOpenMS wheel + if: env.BUILD_PYOPENMS == 'true' + shell: bash + run: | + cd $GITHUB_WORKSPACE/OpenMS/bld/pyOpenMS + python -m pip install wheel + python setup.py bdist_wheel + + - name: Upload pyOpenMS wheel as artifact + if: env.BUILD_PYOPENMS == 'true' + uses: actions/upload-artifact@v4 + with: + name: pyopenms-wheel + path: ${{ github.workspace }}/OpenMS/bld/pyOpenMS/dist/*.whl + - name: Package shell: bash run: | @@ -197,6 +222,13 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} + - name: Download pyOpenMS wheel (if built from source) + if: env.BUILD_PYOPENMS == 'true' + uses: actions/download-artifact@v4 + with: + name: pyopenms-wheel + path: pyopenms-wheel + - name: Setup virtual environment shell: cmd run: | @@ -204,7 +236,12 @@ jobs: call myenv\Scripts\activate.bat - pip install -r requirements.txt + if "${{ env.BUILD_PYOPENMS }}" == "true" ( + pip install pyopenms-wheel/*.whl + pip install -r requirements.txt + ) else ( + pip install -r requirements.txt + ) pip install pyinstaller diff --git a/.github/workflows/build-windows-executable-app.yaml b/.github/workflows/build-windows-executable-app.yaml index 0bacc1a6..eb417a1b 100644 --- a/.github/workflows/build-windows-executable-app.yaml +++ b/.github/workflows/build-windows-executable-app.yaml @@ -15,6 +15,8 @@ permissions: env: OPENMS_VERSION: 3.2.0 PYTHON_VERSION: 3.11.0 + # Control whether to build pyOpenMS from source (true) or use pip (false). Default: false + BUILD_PYOPENMS: false # Name of the installer APP_NAME: OpenMS-StreamlitTemplateApp # Define unique GUID for UpgradeCode @@ -150,6 +152,7 @@ jobs: BUILD_TYPE: "Release" OPENMP: "Off" USE_STATIC_BOOST: "On" + PYOPENMS: "${{ env.BUILD_PYOPENMS == 'true' && 'ON' || 'OFF' }}" # BUILD_FLAGS: "-p:CL_MPCount=2" # For VS Generator and MSBuild BUILD_FLAGS: "-j2" # Ninja will otherwise use all cores (doesn't go well in GHA) CMAKE_CCACHE_EXE: "ccache" @@ -159,6 +162,28 @@ jobs: CCACHE_COMPRESSLEVEL: 12 CCACHE_MAXSIZE: 400M + - name: Build pyOpenMS + if: env.BUILD_PYOPENMS == 'true' + shell: bash + run: | + cd $GITHUB_WORKSPACE/OpenMS/bld/ + ninja pyopenms + + - name: Package pyOpenMS wheel + if: env.BUILD_PYOPENMS == 'true' + shell: bash + run: | + cd $GITHUB_WORKSPACE/OpenMS/bld/pyOpenMS + python -m pip install wheel + python setup.py bdist_wheel + + - name: Upload pyOpenMS wheel as artifact + if: env.BUILD_PYOPENMS == 'true' + uses: actions/upload-artifact@v4 + with: + name: pyopenms-wheel + path: ${{ github.workspace }}/OpenMS/bld/pyOpenMS/dist/*.whl + - name: Test Windows shell: bash run: ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/citest.cmake @@ -251,8 +276,26 @@ jobs: run: | sed -i 's/#import site/import site/' python-${{ env.PYTHON_VERSION }}/python311._pth - - name: Install Required Packages - run: .\python-${{ env.PYTHON_VERSION }}\python -m pip install --force-reinstall -r requirements.txt --no-warn-script-location + - name: Download pyOpenMS wheel (if built from source) + if: env.BUILD_PYOPENMS == 'true' + uses: actions/download-artifact@v4 + with: + name: pyopenms-wheel + path: pyopenms-wheel + + - name: Install pyOpenMS from wheel (if built from source) + if: env.BUILD_PYOPENMS == 'true' + run: | + .\python-${{ env.PYTHON_VERSION }}\python -m pip install pyopenms-wheel/*.whl --no-warn-script-location + + - name: Install Required Packages (excluding pyOpenMS if built from source) + run: | + if ("${{ env.BUILD_PYOPENMS }}" -eq "true") { + Get-Content requirements.txt | Where-Object { $_ -notmatch '^pyopenms' } | Set-Content requirements_filtered.txt + .\python-${{ env.PYTHON_VERSION }}\python -m pip install --force-reinstall -r requirements_filtered.txt --no-warn-script-location + } else { + .\python-${{ env.PYTHON_VERSION }}\python -m pip install --force-reinstall -r requirements.txt --no-warn-script-location + } - name: Set to offline deployment run: | diff --git a/Dockerfile b/Dockerfile index 380508c8..bad50e8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ # It also adds a basic streamlit server that serves a pyOpenMS-based app. # hints: # build image and give it a name (here: streamlitapp) with: docker build --no-cache -t streamlitapp:latest --build-arg GITHUB_TOKEN= . 2>&1 | tee build.log +# To install pyOpenMS from conda instead of building from source: docker build --no-cache -t streamlitapp:latest --build-arg BUILD_PYOPENMS=OFF --build-arg GITHUB_TOKEN= . 2>&1 | tee build.log # check if image was build: docker image ls # run container: docker run -p 8501:8501 streamlitappsimple:latest # debug container after build (comment out ENTRYPOINT) and run container with interactive /bin/bash shell @@ -11,6 +12,8 @@ FROM ubuntu:22.04 AS setup-build-system ARG OPENMS_REPO=https://github.com/OpenMS/OpenMS.git ARG OPENMS_BRANCH=release/3.4.1 ARG PORT=8501 +# Control whether to build pyOpenMS from source (ON) or install from conda (OFF). Default: ON +ARG BUILD_PYOPENMS=ON # GitHub token to download latest OpenMS executable for Windows from Github action artifact. ARG GITHUB_TOKEN ENV GH_TOKEN=${GITHUB_TOKEN} @@ -56,65 +59,82 @@ RUN echo "mamba activate streamlit-env" >> ~/.bashrc SHELL ["/bin/bash", "--rcfile", "~/.bashrc"] SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] -# Install up-to-date cmake via mamba and packages for pyOpenMS build. +# Install up-to-date cmake via mamba and packages for pyOpenMS build (if building from source). RUN mamba install cmake -RUN pip install --upgrade pip && python -m pip install -U setuptools nose 'cython<3.1' 'autowrap<0.23' pandas numpy pytest +ARG BUILD_PYOPENMS +RUN if [ "$BUILD_PYOPENMS" = "ON" ]; then \ + pip install --upgrade pip && python -m pip install -U setuptools nose 'cython<3.1' 'autowrap<0.23' pandas numpy pytest; \ + fi # Clone OpenMS branch and the associcated contrib+thirdparties+pyOpenMS-doc submodules. -RUN git clone --recursive --depth=1 -b ${OPENMS_BRANCH} --single-branch ${OPENMS_REPO} && cd /OpenMS +RUN if [ "$BUILD_PYOPENMS" = "ON" ]; then \ + git clone --recursive --depth=1 -b ${OPENMS_BRANCH} --single-branch ${OPENMS_REPO} && cd /OpenMS; \ + fi # Pull Linux compatible third-party dependencies and store them in directory thirdparty. WORKDIR /OpenMS -RUN mkdir /thirdparty && \ - git submodule update --init THIRDPARTY && \ - cp -r THIRDPARTY/All/* /thirdparty && \ - cp -r THIRDPARTY/Linux/x86_64/* /thirdparty && \ - chmod -R +x /thirdparty +ARG BUILD_PYOPENMS +RUN if [ "$BUILD_PYOPENMS" = "ON" ]; then \ + mkdir /thirdparty && \ + git submodule update --init THIRDPARTY && \ + cp -r THIRDPARTY/All/* /thirdparty && \ + cp -r THIRDPARTY/Linux/x86_64/* /thirdparty && \ + chmod -R +x /thirdparty; \ + fi ENV PATH="/thirdparty/LuciPHOr2:/thirdparty/MSGFPlus:/thirdparty/Sirius:/thirdparty/ThermoRawFileParser:/thirdparty/Comet:/thirdparty/Fido:/thirdparty/MaRaCluster:/thirdparty/MyriMatch:/thirdparty/OMSSA:/thirdparty/Percolator:/thirdparty/SpectraST:/thirdparty/XTandem:/thirdparty/crux:${PATH}" # Build OpenMS and pyOpenMS. FROM setup-build-system AS compile-openms WORKDIR / -# Set up build directory. -RUN mkdir /openms-build -WORKDIR /openms-build - -# Configure. -RUN /bin/bash -c "cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF -DPYOPENMS=ON ../OpenMS -DPY_MEMLEAK_DISABLE=On" - -# Build TOPP tools and clean up. -RUN make -j4 TOPP -RUN rm -rf src doc CMakeFiles - -# Build pyOpenMS wheels and install via pip. -RUN make -j4 pyopenms -WORKDIR /openms-build/pyOpenMS -RUN pip install dist/*.whl +ARG BUILD_PYOPENMS +# Set up build directory and build pyOpenMS from source if enabled. +RUN if [ "$BUILD_PYOPENMS" = "ON" ]; then \ + mkdir /openms-build && \ + cd /openms-build && \ + /bin/bash -c "cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF -DPYOPENMS=ON ../OpenMS -DPY_MEMLEAK_DISABLE=On" && \ + make -j4 TOPP && \ + rm -rf src doc CMakeFiles && \ + make -j4 pyopenms && \ + cd /openms-build/pyOpenMS && \ + pip install dist/*.whl; \ + fi -# Install other dependencies (excluding pyopenms) +# Install dependencies. COPY requirements.txt ./requirements.txt -RUN grep -Ev '^pyopenms([=<>!~].*)?$' requirements.txt > requirements_cleaned.txt && mv requirements_cleaned.txt requirements.txt +RUN if [ "$BUILD_PYOPENMS" = "ON" ]; then \ + # If building from source, exclude pyopenms from requirements.txt \ + grep -Ev '^pyopenms([=<>!~].*)?$' requirements.txt > requirements_cleaned.txt && mv requirements_cleaned.txt requirements.txt; \ + fi RUN pip install -r requirements.txt WORKDIR / RUN mkdir openms -# Copy TOPP tools bin directory, add to PATH. -RUN cp -r openms-build/bin /openms/bin +ARG BUILD_PYOPENMS +# Copy TOPP tools bin directory, add to PATH (only if built from source). +RUN if [ "$BUILD_PYOPENMS" = "ON" ]; then \ + cp -r openms-build/bin /openms/bin; \ + fi ENV PATH="/openms/bin/:${PATH}" -# Copy TOPP tools bin directory, add to PATH. -RUN cp -r openms-build/lib /openms/lib +# Copy TOPP tools lib directory, add to LD_LIBRARY_PATH (only if built from source). +RUN if [ "$BUILD_PYOPENMS" = "ON" ]; then \ + cp -r openms-build/lib /openms/lib; \ + fi ENV LD_LIBRARY_PATH="/openms/lib/:${LD_LIBRARY_PATH}" -# Copy share folder, add to PATH, remove source directory. -RUN cp -r OpenMS/share/OpenMS /openms/share -RUN rm -rf OpenMS +# Copy share folder, add to PATH, remove source directory (only if built from source). +RUN if [ "$BUILD_PYOPENMS" = "ON" ]; then \ + cp -r OpenMS/share/OpenMS /openms/share && \ + rm -rf OpenMS; \ + fi ENV OPENMS_DATA_PATH="/openms/share/" -# Remove build directory. -RUN rm -rf openms-build +# Remove build directory (only if built from source). +RUN if [ "$BUILD_PYOPENMS" = "ON" ]; then \ + rm -rf openms-build; \ + fi # Prepare and run streamlit app. FROM compile-openms AS run-app From 12bff57ce9b858b6ca0ac801d08999a21fbac903 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 13:48:58 +0000 Subject: [PATCH 3/5] Add Python setup and dependencies for pyOpenMS build in Windows workflows Co-authored-by: t0mdavid-m <57191390+t0mdavid-m@users.noreply.github.com> --- ...d-windows-executable-app-with-pyinstaller.yaml | 15 +++++++++++++++ .../workflows/build-windows-executable-app.yaml | 13 +++++++++++++ 2 files changed, 28 insertions(+) diff --git a/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml b/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml index 2958b2cd..30a722a4 100644 --- a/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml +++ b/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml @@ -11,6 +11,8 @@ env: jobs: build-openms: runs-on: windows-latest + env: + PYTHON_VERSION: 3.11.0 steps: - name: Checkout @@ -55,6 +57,19 @@ jobs: # https://github.com/actions/runner-images/blob/main/images/win/scripts/Installers/Install-GitHub-CLI.ps1 echo "C:/Program Files (x86)/GitHub CLI" >> $GITHUB_PATH + - name: Set up Python for pyOpenMS build + if: env.BUILD_PYOPENMS == 'true' + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Python dependencies for pyOpenMS build + if: env.BUILD_PYOPENMS == 'true' + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install setuptools wheel nose 'cython<3.1' 'autowrap<0.23' numpy + - name: Extract branch/PR infos shell: bash run: | diff --git a/.github/workflows/build-windows-executable-app.yaml b/.github/workflows/build-windows-executable-app.yaml index eb417a1b..dd0ddf26 100644 --- a/.github/workflows/build-windows-executable-app.yaml +++ b/.github/workflows/build-windows-executable-app.yaml @@ -72,6 +72,19 @@ jobs: # https://github.com/actions/runner-images/blob/main/images/win/scripts/Installers/Install-GitHub-CLI.ps1 echo "C:/Program Files (x86)/GitHub CLI" >> $GITHUB_PATH + - name: Set up Python for pyOpenMS build + if: env.BUILD_PYOPENMS == 'true' + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Python dependencies for pyOpenMS build + if: env.BUILD_PYOPENMS == 'true' + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install setuptools wheel nose 'cython<3.1' 'autowrap<0.23' numpy + - name: Extract branch/PR infos shell: bash run: | From c4a79a932000b00e57f50e6b36078d5f2a9dbda3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 13:51:26 +0000 Subject: [PATCH 4/5] Fix code review issues: update Python version, fix regex patterns, filter requirements.txt properly Co-authored-by: t0mdavid-m <57191390+t0mdavid-m@users.noreply.github.com> --- .../build-windows-executable-app-with-pyinstaller.yaml | 5 +++-- .github/workflows/build-windows-executable-app.yaml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml b/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml index 30a722a4..882fd277 100644 --- a/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml +++ b/.github/workflows/build-windows-executable-app-with-pyinstaller.yaml @@ -59,7 +59,7 @@ jobs: - name: Set up Python for pyOpenMS build if: env.BUILD_PYOPENMS == 'true' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -253,7 +253,8 @@ jobs: if "${{ env.BUILD_PYOPENMS }}" == "true" ( pip install pyopenms-wheel/*.whl - pip install -r requirements.txt + findstr /v /r /c:"^pyopenms" requirements.txt > requirements_filtered.txt + pip install -r requirements_filtered.txt ) else ( pip install -r requirements.txt ) diff --git a/.github/workflows/build-windows-executable-app.yaml b/.github/workflows/build-windows-executable-app.yaml index dd0ddf26..d2510372 100644 --- a/.github/workflows/build-windows-executable-app.yaml +++ b/.github/workflows/build-windows-executable-app.yaml @@ -304,7 +304,7 @@ jobs: - name: Install Required Packages (excluding pyOpenMS if built from source) run: | if ("${{ env.BUILD_PYOPENMS }}" -eq "true") { - Get-Content requirements.txt | Where-Object { $_ -notmatch '^pyopenms' } | Set-Content requirements_filtered.txt + Get-Content requirements.txt | Where-Object { $_ -notmatch '^pyopenms([=<>!~\s]|$)' } | Set-Content requirements_filtered.txt .\python-${{ env.PYTHON_VERSION }}\python -m pip install --force-reinstall -r requirements_filtered.txt --no-warn-script-location } else { .\python-${{ env.PYTHON_VERSION }}\python -m pip install --force-reinstall -r requirements.txt --no-warn-script-location From 29e1b2d914b60bc8aef7bd391907a354152c8564 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 13:53:33 +0000 Subject: [PATCH 5/5] Add documentation for BUILD_PYOPENMS configuration option Co-authored-by: t0mdavid-m <57191390+t0mdavid-m@users.noreply.github.com> --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 3b995b66..a1b65d26 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ To run the app locally: This repository contains two Dockerfiles. 1. `Dockerfile`: This Dockerfile builds all dependencies for the app including Python packages and the OpenMS TOPP tools. Recommended for more complex workflows where you want to use the OpenMS TOPP tools for instance with the **TOPP Workflow Framework**. + - By default, it builds pyOpenMS from source. To install pyOpenMS from conda instead, use `--build-arg BUILD_PYOPENMS=OFF`. 2. `Dockerfile_simple`: This Dockerfile builds only the Python packages. Recommended for simple apps using pyOpenMS only. 1. **Install Docker** @@ -116,6 +117,23 @@ This repository contains two Dockerfiles. docker run -p 8505:8501 openms_streamlit_template ``` +## ⚙️ Build Configuration + +### pyOpenMS Build Options + +You can control whether pyOpenMS is built from source or installed from conda/pip: + +#### Docker +- **Build from source (default)**: `docker build -t streamlitapp:latest .` +- **Install from conda**: `docker build --build-arg BUILD_PYOPENMS=OFF -t streamlitapp:latest .` + +#### Windows Workflows +The GitHub Actions workflows (`build-windows-executable-app.yaml` and `build-windows-executable-app-with-pyinstaller.yaml`) can be configured via the `BUILD_PYOPENMS` environment variable: +- **Install from pip (default)**: `BUILD_PYOPENMS: false` +- **Build from source**: `BUILD_PYOPENMS: true` + +Change this value in the workflow file under the `env:` section to enable building pyOpenMS from source. + ## Documentation Documentation for **users** and **developers** is included as pages in [this template app](https://abi-services.cs.uni-tuebingen.de/streamlit-template/), indicated by the 📖 icon.