Merge branch 'main' into col-1214

Author: kosiew

.github/workflows/build.yml

@@ -48,6 +48,10 @@ jobs:
           uv run --no-project ruff check --output-format=github python/
           uv run --no-project ruff format --check python/
 
+      - name: Run codespell
+        run: |
+            uv run --no-project codespell --toml pyproject.toml
+
   generate-license:
     runs-on: ubuntu-latest
     steps:
@@ -271,7 +275,100 @@ jobs:
         with:
           name: dist
           pattern: dist-*
-  
+
+  # Documentation build job that runs after wheels are built
+  build-docs:
+    name: Build docs
+    runs-on: ubuntu-latest
+    needs: [build-manylinux-x86_64]  # Only need the Linux wheel for docs
+    # Only run docs on main branch pushes, tags, or PRs
+    if: github.event_name == 'push' || github.event_name == 'pull_request'
+    steps:
+      - name: Set target branch
+        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag')
+        id: target-branch
+        run: |
+          set -x
+          if test '${{ github.ref }}' = 'refs/heads/main'; then
+            echo "value=asf-staging" >> "$GITHUB_OUTPUT"
+          elif test '${{ github.ref_type }}' = 'tag'; then
+            echo "value=asf-site" >> "$GITHUB_OUTPUT"
+          else
+            echo "Unsupported input: ${{ github.ref }} / ${{ github.ref_type }}"
+            exit 1
+          fi
+
+      - name: Checkout docs sources
+        uses: actions/checkout@v5
+
+      - name: Checkout docs target branch
+        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag')
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          ref: ${{ steps.target-branch.outputs.value }}
+          path: docs-target
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      # Download the Linux wheel built in the previous job
+      - name: Download pre-built Linux wheel
+        uses: actions/download-artifact@v5
+        with:
+          name: dist-manylinux-x86_64
+          path: wheels/
+
+      # Install from the pre-built wheel
+      - name: Install from pre-built wheel
+        run: |
+          set -x
+          uv venv
+          # Install documentation dependencies
+          uv sync --dev --no-install-package datafusion --group docs
+          # Install the pre-built wheel
+          WHEEL=$(find wheels/ -name "*.whl" | head -1)
+          if [ -n "$WHEEL" ]; then
+            echo "Installing wheel: $WHEEL"
+            uv pip install "$WHEEL"
+          else
+            echo "ERROR: No wheel found!"
+            exit 1
+          fi
+
+      - name: Build docs
+        run: |
+          set -x
+          cd docs
+          curl -O https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv
+          curl -O https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet
+          uv run --no-project make html
+
+      - name: Copy & push the generated HTML
+        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag')
+        run: |
+          set -x
+          cd docs-target
+          # delete anything but: 1) '.'; 2) '..'; 3) .git/
+          find ./ | grep -vE "^./$|^../$|^./.git" | xargs rm -rf
+          cp ../.asf.yaml .
+          cp -r ../docs/build/html/* .
+          git status --porcelain
+          if [ "$(git status --porcelain)" != "" ]; then
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+            git add --all
+            git commit -m 'Publish built docs triggered by ${{ github.sha }}'
+            git push || git push --force
+          fi
+
   # NOTE: PyPI publish needs to be done manually for now after release passed the vote
   # release:
   #   name: Publish in PyPI

.github/workflows/docs.yaml

@@ -80,13 +80,6 @@ jobs:
         with:
             enable-cache: true
 
-      - name: Check documentation
-        if: ${{ matrix.python-version == '3.10' && matrix.toolchain == 'stable' }}
-        run: |
-          uv sync --dev --group docs --no-install-package datafusion
-          uv run --no-project maturin develop --uv
-          uv run --no-project docs/build.sh
-
       - name: Run tests
         env:
           RUST_BACKTRACE: 1

.github/workflows/test.yaml

@@ -45,5 +45,13 @@ repos:
             types: [file, rust]
             language: system
 
+      - repo: https://github.com/codespell-project/codespell
+        rev: v2.4.1
+        hooks:
+          - id: codespell
+            args: [ --toml, "pyproject.toml"]
+            additional_dependencies:
+              - tomli
+
 default_language_version:
       python: python3

.pre-commit-config.yaml

@@ -233,7 +233,7 @@ and for `uv run` commands the additional parameter `--no-project`
 git clone git@github.com:apache/datafusion-python.git
 # cd to the repo root
 cd datafusion-python/
-# create the virtual enviornment
+# create the virtual environment
 uv sync --dev --no-install-package datafusion
 # activate the environment
 source .venv/bin/activate

README.md

@@ -35,4 +35,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) --fail-on-warning

docs/Makefile

@@ -195,7 +195,7 @@ optimization levels. If you wish to go down this route, there are two approaches
 have identified you can use.
 
 #. Re-export all of ``datafusion-python`` yourself with your extensions built in.
-#. Carefully synchonize your software releases with the ``datafusion-python`` CI build
+#. Carefully synchronize your software releases with the ``datafusion-python`` CI build
    system so that your libraries use the exact same compiler, features, and
    optimization level.
 

docs/source/contributor-guide/ffi.rst

@@ -43,7 +43,7 @@ Bootstrap:
 
     # fetch this repo
     git clone git@github.com:apache/datafusion-python.git
-    # create the virtual enviornment
+    # create the virtual environment
     uv sync --dev --no-install-package datafusion
     # activate the environment
     source .venv/bin/activate

docs/source/contributor-guide/introduction.rst

@@ -64,7 +64,7 @@ Arrays
 ------
 
 For columns that contain arrays of values, you can access individual elements of the array by index
-using bracket indexing. This is similar to callling the function
+using bracket indexing. This is similar to calling the function
 :py:func:`datafusion.functions.array_element`, except that array indexing using brackets is 0 based,
 similar to Python arrays and ``array_element`` is 1 based indexing to be compatible with other SQL
 approaches.
@@ -82,6 +82,13 @@ approaches.
     Indexing an element of an array via ``[]`` starts at index 0 whereas
     :py:func:`~datafusion.functions.array_element` starts at index 1.
 
+Starting in DataFusion 49.0.0 you can also create slices of array elements using
+slice syntax from Python.
+
+.. ipython:: python
+
+    df.select(col("a")[1:3].alias("second_two_elements"))
+
 To check if an array is empty, you can use the function :py:func:`datafusion.functions.array_empty` or `datafusion.functions.empty`.
 This function returns a boolean indicating whether the array is empty.
 

docs/source/user-guide/common-operations/expressions.rst

@@ -24,7 +24,7 @@ In this section you will learn about window functions. A window function utilize
 multiple rows to produce a result for each individual row, unlike an aggregate function that
 provides a single value for multiple rows.
 
-The window functions are availble in the :py:mod:`~datafusion.functions` module.
+The window functions are available in the :py:mod:`~datafusion.functions` module.
 
 We'll use the pokemon dataset (from Ritchie Vink) in the following examples.
 
@@ -99,8 +99,8 @@ If you do not specify a Window Frame, the frame will be set depending on the fol
 criteria.
 
 * If an ``order_by`` clause is set, the default window frame is defined as the rows between
-  unbounded preceeding and the current row.
-* If an ``order_by`` is not set, the default frame is defined as the rows betwene unbounded
+  unbounded preceding and the current row.
+* If an ``order_by`` is not set, the default frame is defined as the rows between unbounded
   and unbounded following (the entire partition).
 
 Window Frames are defined by three parameters: unit type, starting bound, and ending bound.
@@ -116,7 +116,7 @@ The unit types available are:
   ``order_by`` clause.
 
 In this example we perform a "rolling average" of the speed of the current Pokemon and the
-two preceeding rows.
+two preceding rows.
 
 .. ipython:: python