diff --git a/.github/actions/run-smoke/action.yml b/.github/actions/run-smoke/action.yml index 8c73603d..ccfc37cd 100644 --- a/.github/actions/run-smoke/action.yml +++ b/.github/actions/run-smoke/action.yml @@ -22,14 +22,19 @@ runs: export PATH="${PATH}:${CUDA_HOME}/bin" nvidia-smi + - name: "Install uv" + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + # installs in $GITHUB_WORKSPACE/venv. # only has to install Tox because Tox will do the other virtual environment management. - name: "Setup Python virtual environment" shell: bash run: | - python${{ inputs.python-version }} -m venv --upgrade-deps venv + uv venv venv --python python${{ inputs.python-version }} . venv/bin/activate - pip install tox -c constraints-dev.txt + uv pip install tox tox-uv -c constraints-dev.txt # flash-attn has a bug in the setup.py that causes pip to attempt # installing it before torch is installed. This is a bug because their @@ -41,22 +46,15 @@ runs: run: | source venv/bin/activate # The list is taken from the pull request linked above - pip install torch packaging setuptools wheel psutil ninja -c constraints-dev.txt - - - name: "Install tox-current-env to reuse the venv with pre-installed build dependencies" - shell: bash - run: | - source venv/bin/activate - pip install tox-current-env + uv pip install torch packaging setuptools wheel psutil ninja -c constraints-dev.txt - - name: "Install dependencies from tox.ini in the current venv, using current venv installed deps" + - name: "Install dependencies from tox.ini in the current venv" shell: bash run: | source venv/bin/activate tox -e py3-smoke --print-deps-to-file=./deps.txt - pip_install="pip install -c constraints-dev.txt" - $pip_install -r ./deps.txt --no-build-isolation - $pip_install . + uv pip install -c constraints-dev.txt -r ./deps.txt --no-build-isolation + uv pip install -c constraints-dev.txt . - name: "Show disk utilization BEFORE tests" shell: bash @@ -68,7 +66,7 @@ runs: shell: bash run: | source venv/bin/activate - tox --current-env -e py3-smoke + tox -e py3-smoke - name: "Show disk utilization AFTER tests" shell: bash diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2c8df16d..df8fc719 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -69,18 +69,21 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: 3.11 - cache: pip - cache-dependency-path: | - **/pyproject.toml - **/requirements*.txt + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true - name: Install tox run: | - python -m pip install --upgrade pip - python -m pip install tox tox-gh -c constraints-dev.txt + uv venv venv + source venv/bin/activate + uv pip install tox tox-uv -c constraints-dev.txt - name: "${{ matrix.lint.name }}" run: | + source venv/bin/activate ${{ matrix.lint.commands }} env: RUFF_OUTPUT_FORMAT: github diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml index c317cafd..395793fd 100644 --- a/.github/workflows/unit.yaml +++ b/.github/workflows/unit.yaml @@ -62,13 +62,18 @@ jobs: with: fetch-depth: 0 + - name: "Install uv" + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + # installs in $GITHUB_WORKSPACE/venv. # only has to install Tox because Tox will do the other virtual environment management. - name: "Setup Python virtual environment" run: | - python -m venv --upgrade-deps venv + uv venv venv . venv/bin/activate - pip install tox -c constraints-dev.txt + uv pip install tox tox-uv -c constraints-dev.txt - name: "Show disk utilization BEFORE tests" if: always() diff --git a/constraints-dev.txt b/constraints-dev.txt index 26dce39e..33c66013 100644 --- a/constraints-dev.txt +++ b/constraints-dev.txt @@ -185,8 +185,8 @@ tokenizers==0.22.0 # via transformers tomlkit==0.13.3 # via pylint torch==2.6.0 # via accelerate, bitsandbytes, deepspeed, flash-attn, liger-kernel, peft, -c constraints-dev.txt.in, -r requirements.txt tornado==6.5.2 # via ipykernel, jupyter-client, jupyter-server, jupyterlab, notebook, terminado -tox==4.29.0 # via tox-current-env, -r requirements-dev.txt -tox-current-env==0.0.16 # via -r requirements-dev.txt +tox==4.29.0 # via tox-uv, -r requirements-dev.txt +tox-uv==1.25.0 # via -r requirements-dev.txt tqdm==4.67.1 # via datasets, deepspeed, huggingface-hub, peft, transformers traitlets==5.14.3 # via ipykernel, ipython, ipywidgets, jupyter-client, jupyter-console, jupyter-core, jupyter-events, jupyter-server, jupyterlab, matplotlib-inline, nbclient, nbconvert, nbformat transformers==4.56.0 # via peft, trl, -r requirements.txt diff --git a/requirements-dev.txt b/requirements-dev.txt index 3fb79e43..c7b14e49 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,8 +10,8 @@ pylint>=2.16.2 pylint-pydantic pytest ruff -tox>=4.4.2 -tox-current-env +tox>=4 +tox-uv mypy>=1.10.0 types-tqdm diff --git a/tox.ini b/tox.ini index 0794c417..401be6b0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 [tox] +requires = + tox>=4 + tox-uv envlist = ruff, lint, mypy, py3-unit minversion = 4.4 @@ -8,18 +11,27 @@ minversion = 4.4 description = run tests package = wheel wheel_build_env = pkg + +# Use uv for package installation and management +runner = uv-venv-runner +uv_seed = true + deps = pytest -install_command = pip install \ +install_command = uv pip install \ -c constraints-dev.txt \ {opts} {packages} +passenv = + PATH [testenv:py3] basepython = python3.11 [testenv:py3-unit] description = run unit tests with pytest +basepython = python3.11 passenv = + PATH HF_HOME INSTRUCTLAB_NCCL_TIMEOUT_MS CMAKE_ARGS @@ -39,9 +51,11 @@ commands = {envpython} -m pytest tests/unit {posargs} [testenv:py3-smoke] description = run accelerated smoke tests with pytest +basepython = python3.11 passenv = + PATH HF_HOME - INSTRUCTLAB_NCCL_TIMEOUT_MS + INSTRUCTLAB_NCCL_TIMEOUT_MS deps = -r requirements-dev.txt -r requirements-cuda.txt