From 1d928395569e747e8c8bac9c6f3f4b0877182022 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Jan 2026 19:06:22 +0000 Subject: [PATCH] Consolidate documentation and add SessionStart hook for AI assistants - Add SessionStart hook (.claude/hooks/SessionStart.sh) to auto-verify environment setup - Consolidate AGENTS.md content into docs/reference/contributing.md (single source of truth) - Enhance README.md with better examples, features, use cases, and community links - Update docs/index.md to include README content (eliminates duplication) - Remove .github/AGENTS.md (no longer needed) Benefits: - No duplication between files - easier maintenance - SessionStart hook ensures proper setup for Claude Code sessions - Contributing.md is now comprehensive guide for both humans and AI - README provides rich landing page while feeding docs homepage - Tool-agnostic approach works for all AI assistants --- .claude/hooks/SessionStart.sh | 51 +++++++ .github/AGENTS.md | 205 ------------------------- README.md | 127 ++++++++++++++- docs/index.md | 63 +------- docs/reference/contributing.md | 272 ++++++++++++++++++++++++++++++++- 5 files changed, 444 insertions(+), 274 deletions(-) create mode 100755 .claude/hooks/SessionStart.sh delete mode 100644 .github/AGENTS.md diff --git a/.claude/hooks/SessionStart.sh b/.claude/hooks/SessionStart.sh new file mode 100755 index 00000000..4f5fc204 --- /dev/null +++ b/.claude/hooks/SessionStart.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e + +echo "πŸš€ Setting up egglog-python development environment..." +echo "" + +# Check if uv is installed +if ! command -v uv &> /dev/null; then + echo "❌ Error: uv is not installed" + echo " Install with: curl -LsSf https://astral.sh/uv/install.sh | sh" + exit 1 +fi + +echo "πŸ“¦ Syncing dependencies..." +if uv sync --all-extras --locked 2>&1 | grep -q "Resolved"; then + echo "βœ… Dependencies synced successfully" +else + echo "βœ… Dependencies already up to date" +fi + +echo "" +echo "πŸ” Running quick validation checks..." +echo "" + +# Run ruff check (non-blocking) +echo " β†’ Checking code style with ruff..." +if uv run ruff check . --quiet 2>&1; then + echo " βœ… Ruff checks passed" +else + echo " ⚠️ Ruff found some issues (run 'uv run ruff check --fix .' to auto-fix)" +fi + +# Run quick type check (non-blocking) +echo " β†’ Type checking with mypy..." +if make mypy 2>&1 | tail -n 1 | grep -q "Success"; then + echo " βœ… Type checks passed" +else + echo " ⚠️ Type check issues found (run 'make mypy' for details)" +fi + +echo "" +echo "✨ Environment ready! Quick reference:" +echo "" +echo " Run tests: uv run pytest --benchmark-disable -vvv" +echo " After Rust edit: uv sync --reinstall-package egglog --all-extras" +echo " Format code: uv run ruff format ." +echo " Fix linting: uv run ruff check --fix ." +echo " Type check: make mypy" +echo "" +echo "πŸ“š See docs/reference/contributing.md for complete development guide" +echo "" diff --git a/.github/AGENTS.md b/.github/AGENTS.md deleted file mode 100644 index 1e0f18d8..00000000 --- a/.github/AGENTS.md +++ /dev/null @@ -1,205 +0,0 @@ -# Agent Instructions for egglog-python - -This file provides instructions for AI coding agents (including GitHub Copilot) working on this repository. - -## Project Overview - -This repository provides Python bindings for the Rust library `egglog`, enabling the use of e-graphs in Python for optimization, symbolic computation, and analysis. It is a hybrid project combining: -- **Python code** in `python/egglog/` - The main Python API and library -- **Rust code** in `src/` - PyO3-based bindings to the egglog Rust library -- **Documentation** in `docs/` - Sphinx-based documentation - -## Repository Structure - -- `python/egglog/` - Main Python package source code -- `python/tests/` - Python test suite (pytest-based) -- `src/` - Rust source code for Python bindings (PyO3) -- `docs/` - Documentation source files (Sphinx) -- `test-data/` - Test data files -- `pyproject.toml` - Python project configuration and dependencies -- `Cargo.toml` - Rust project configuration -- `uv.lock` - Locked dependencies (managed by uv) - -## Build and Development Commands - -### Prerequisites -- **uv** - Package manager (https://github.com/astral-sh/uv) -- **Rust toolchain** - Version pinned in `rust-toolchain.toml` -- **Python** - Version pinned in `.python-version` - -### Common Commands - -```bash -# Install dependencies -uv sync --all-extras - -# Reinstall the Rust extension after changing code in `src/` -uv sync --reinstall-package egglog --all-extras - -# Run tests -uv run pytest --benchmark-disable -vvv --durations=10 - -# Type checking with mypy -make mypy - -# Stub testing -make stubtest - -# Build documentation -make docs - -# Refresh the bundled visualizer assets -make clean -make - -# Format code (auto-run by pre-commit) -uv run ruff format . - -# Lint code (auto-run by pre-commit) -uv run ruff check --fix . -``` - -## Python Code Standards - -### General Guidelines -- **Line length**: 120 characters maximum -- **Type hints**: Use type annotations for public APIs and functions -- **Formatting**: Use Ruff for code formatting and linting -- **Testing**: Write tests using pytest in `python/tests/` -- **Docstrings**: Use clear, concise docstrings for public functions and classes - -### Ruff Configuration -The project uses Ruff for linting and formatting with specific rules: -- Allows uppercase variable names (N806, N802) -- Allows star imports (F405, F403) -- Allows `exec` and subprocess usage (S102, S307, S603) -- Allows `Any` type annotations (ANN401) -- Test files don't require full type annotations - -See `pyproject.toml` for complete Ruff configuration. - -### Type Checking -- **mypy** is used for static type checking -- Run `make mypy` to type check Python code -- Run `make stubtest` to validate type stubs against runtime behavior -- Exclusions: `__snapshots__`, `_build`, `conftest.py` - -### Testing -- Tests are located in `python/tests/` -- Use pytest with snapshot testing (syrupy) -- Benchmarks use pytest-benchmark and CodSpeed -- Run tests with: `uv run pytest --benchmark-disable -vvv` - -## Rust Code Standards - -### General Guidelines -- **Edition**: Rust 2024 (experimental) -- **FFI**: Uses PyO3 for Python bindings -- **Main library**: Uses egglog from git (saulshanabrook/egg-smol, clone-cost branch) - -### Rust File Organization -- `src/lib.rs` - Main library entry point -- `src/egraph.rs` - E-graph implementation -- `src/conversions.rs` - Type conversions between Python and Rust -- `src/py_object_sort.rs` - Python object handling -- `src/extract.rs` - Extraction functionality -- `src/error.rs` - Error handling -- `src/serialize.rs` - Serialization support -- `src/termdag.rs` - Term DAG operations -- `src/utils.rs` - Utility functions - -### Python File Organization - -#### Public Interface -All public Python APIs are exported from the top-level `egglog` module. Anything that is public should be exported in `python/egglog/__init__.py` at the top level. - -#### Lower-Level Bindings -The `egglog.bindings` module provides lower-level access to the Rust implementation for advanced use cases. - -#### Core Python Files -- `python/egglog/__init__.py` - Top-level module exports, defines the public API -- `python/egglog/egraph.py` - Main EGraph class and e-graph management -- `python/egglog/egraph_state.py` - E-graph state and execution management -- `python/egglog/runtime.py` - Runtime system for expression evaluation and method definitions -- `python/egglog/builtins.py` - Built-in types (i64, f64, String, Vec, etc.) and operations -- `python/egglog/declarations.py` - Class, function, and method declaration decorators -- `python/egglog/conversion.py` - Type conversion between Python and egglog types -- `python/egglog/pretty.py` - Pretty printing for expressions and e-graph visualization -- `python/egglog/deconstruct.py` - Deconstruction of Python values into egglog expressions -- `python/egglog/thunk.py` - Lazy evaluation support -- `python/egglog/type_constraint_solver.py` - Type inference and constraint solving -- `python/egglog/config.py` - Configuration settings -- `python/egglog/ipython_magic.py` - IPython/Jupyter integration -- `python/egglog/visualizer_widget.py` - Interactive visualization widget -- `python/egglog/version_compat.py` - Python version compatibility utilities -- `python/egglog/examples/` - End-to-end samples and tutorials demonstrating the API -- `python/egglog/exp/` - Experimental Array API integrations and code generation helpers - -The compiled extension artifact `python/egglog/bindings.cpython-*.so` is generated by `uv sync` and should not be edited manually. - -## Code Style Preferences - -1. **Imports**: Follow Ruff's import sorting -2. **Naming**: - - Python: snake_case for functions and variables, PascalCase for classes - - Rust: Follow standard Rust conventions -3. **Comments**: Use clear, explanatory comments for complex logic -4. **Documentation**: Keep docs synchronized with code changes - -## Contributing Guidelines - -When making changes: -1. Update or add tests in `python/tests/` for Python changes -2. Run the full test suite before committing -3. Ensure type checking passes with `make mypy` -4. Build documentation if changing public APIs -5. Follow existing code patterns and style -6. Keep changes minimal and focused -7. Ensure the automatic changelog entry in `docs/changelog.md` (added when opening the PR) accurately reflects your change and add manual notes if additional clarification is needed - -## Common Patterns - -### Python API Design -- Define e-graph classes by inheriting from `egglog.Expr` -- Use `@egraph.function` decorator for functions -- Use `@egraph.method` decorator for methods -- Leverage type annotations for better IDE support - -### Working with Values -- Use `get_literal_value(expr)` or the `.value` property to get Python values from primitives -- Use pattern matching with `match`/`case` for destructuring egglog primitives -- Use `get_callable_fn(expr)` to get the underlying Python function from a callable expression -- Use `get_callable_args(expr)` to get arguments to a callable - -### Parallelism -- The underlying Rust library uses Rayon for parallelism -- Control worker thread count via `RAYON_NUM_THREADS` environment variable -- Defaults to single thread if not set - -### Rust-Python Integration -- Use PyO3's `#[pyclass]` and `#[pymethods]` macros -- Handle errors with appropriate Python exceptions -- Convert between Rust and Python types in `conversions.rs` - -## Documentation - -Documentation is built with Sphinx: -- Source files in `docs/` -- Build with `make docs` -- Output in `docs/_build/html/` -- Hosted on ReadTheDocs - -## Testing Strategy - -1. **Unit tests**: Test individual functions and classes -2. **Integration tests**: Test complete workflows -3. **Snapshot tests**: Use syrupy for snapshot testing of complex outputs -4. **Benchmarks**: Performance testing with pytest-benchmark and pytest-codspeed -5. **Parallel testing**: Use pytest-xdist for faster test runs -6. **Type checking**: Validate type stubs and annotations - -## Performance Considerations - -- The library uses Rust for performance-critical operations -- Benchmarking is done via CodSpeed for continuous performance monitoring -- Profile with release builds (`cargo build --release`) when needed diff --git a/README.md b/README.md index 93940c80..f9359ff7 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,125 @@ -# `egglog` Python wrapper +# egglog Python [![Documentation Status](https://readthedocs.org/projects/egglog-python/badge/?version=latest)](https://egglog-python.readthedocs.io/latest/?badge=latest) [![Test](https://github.com/egraphs-good/egglog-python/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/egraphs-good/egglog-python/actions/workflows/CI.yml) [![PyPi Package](https://img.shields.io/pypi/v/egglog.svg)](https://pypi.org/project/egglog/) [![License](https://img.shields.io/pypi/l/egglog.svg)](https://pypi.org/project/egglog/) [![Python Versions](https://img.shields.io/pypi/pyversions/egglog.svg)](https://pypi.org/project/egglog/) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/egraphs-good/egglog-python) -`egglog` is a Python package that provides bindings to the Rust library [`egglog`](https://github.com/egraphs-good/egglog/), -allowing you to use e-graphs in Python for optimization, symbolic computation, and analysis. +**egglog** is a Python package that provides bindings to the Rust library [`egglog`](https://github.com/egraphs-good/egglog/), allowing you to use **e-graphs** in Python for optimization, symbolic computation, and program analysis. -Please see the [documentation](https://egglog-python.readthedocs.io/) for more information. +## What are e-graphs? -Come say hello [on the e-graphs Zulip](https://egraphs.zulipchat.com/#narrow/stream/375765-egglog/) or [open an issue](https://github.com/egraphs-good/egglog-python/issues/new/choose)! +E-graphs (equality graphs) are data structures that efficiently represent equivalence classes of expressions. They enable powerful program optimizations through **equality saturation** - a technique that finds optimal expressions by exploring many equivalent representations simultaneously. -## How to cite +The underlying [`egglog`](https://github.com/egraphs-good/egglog) Rust library combines: +- **Datalog**: Efficient incremental reasoning and queries +- **Equality Saturation**: Term rewriting and optimization +- **E-graphs**: Compact representation of equivalent expressions -If you use **egglog-python** in academic work, please cite the paper: +See the paper ["Better Together: Unifying Datalog and Equality Saturation"](https://arxiv.org/abs/2304.04332) for details. + +## Installation + +```shell +pip install egglog +``` + +Requires Python 3.11+ and works on Linux, macOS, and Windows. + +## Quick Example + +Here's how to use egglog to prove that `2 * (x + 3)` is equivalent to `6 + 2 * x` through algebraic rewriting: + +```{code-cell} python +from __future__ import annotations +from egglog import * + + +class Num(Expr): + def __init__(self, value: i64Like) -> None: ... + + @classmethod + def var(cls, name: StringLike) -> Num: ... + + def __add__(self, other: Num) -> Num: ... + + def __mul__(self, other: Num) -> Num: ... + + +egraph = EGraph() + +# Create two expressions +expr1 = egraph.let("expr1", Num(2) * (Num.var("x") + Num(3))) +expr2 = egraph.let("expr2", Num(6) + Num(2) * Num.var("x")) + + +# Define rewrite rules for algebraic simplification +@egraph.register +def _num_rules(a: Num, b: Num, c: Num, i: i64, j: i64): + yield rewrite(a + b).to(b + a) # Commutative + yield rewrite(a * (b + c)).to((a * b) + (a * c)) # Distributive + yield rewrite(Num(i) + Num(j)).to(Num(i + j)) # Constant folding + yield rewrite(Num(i) * Num(j)).to(Num(i * j)) + + +# Run equality saturation +egraph.saturate() + +# Prove the expressions are equivalent +egraph.check(expr1 == expr2) + +# Extract the simplified form +egraph.extract(expr1) +``` + +## Features + +- **Pythonic API**: Natural Python syntax with type hints and decorators +- **High Performance**: Powered by Rust and the battle-tested egglog library +- **Type Safe**: Full type annotations and mypy support +- **Datalog Integration**: Combine e-graphs with relational queries +- **Rich Ecosystem**: Jupyter integration, visualization tools, examples +- **Well Documented**: Comprehensive tutorials, guides, and API reference + +## Use Cases + +egglog-python is useful for: +- **Compiler optimizations**: Optimize IR or DSL programs +- **Symbolic mathematics**: Simplify and manipulate mathematical expressions +- **Program synthesis**: Generate optimal programs from specifications +- **Query optimization**: Optimize database queries or data transformations +- **Theorem proving**: Automated reasoning with equality + +## Documentation + +**[πŸ“š Full Documentation](https://egglog-python.readthedocs.io/)** - Tutorials, guides, and API reference + +Key sections: +- [Tutorials](https://egglog-python.readthedocs.io/latest/tutorials/) - Step-by-step guides +- [How-to Guides](https://egglog-python.readthedocs.io/latest/how-to-guides/) - Task-oriented recipes +- [API Reference](https://egglog-python.readthedocs.io/latest/reference/) - Complete API documentation +- [Examples](https://egglog-python.readthedocs.io/latest/tutorials/examples/) - Real-world usage examples + +## Contributing + +Contributions are welcome! Whether you want to: +- Report a bug or request a feature +- Improve documentation +- Add examples +- Contribute code + +See **[docs/reference/contributing.md](docs/reference/contributing.md)** for: +- Development setup and environment +- Running tests, linters, and type checkers +- Code standards and architecture overview +- Common patterns and troubleshooting + +## Community + +- **[πŸ’¬ Zulip Chat](https://egraphs.zulipchat.com/)** - Join the e-graphs community +- **[πŸ› Issues](https://github.com/egraphs-good/egglog-python/issues)** - Report bugs or request features +- **[πŸ“– Changelog](https://egglog-python.readthedocs.io/latest/reference/changelog.html)** - See what's new + +## Citation + +If you use **egglog-python** in academic work, please cite: ```bibtex @misc{Shanabrook2023EgglogPython, @@ -25,3 +133,8 @@ If you use **egglog-python** in academic work, please cite the paper: url = {https://arxiv.org/abs/2305.04311}, note = {Presented at EGRAPHS@PLDI 2023} } +``` + +## License + +MIT License - see LICENSE file for details. diff --git a/docs/index.md b/docs/index.md index cce796be..8360676a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,66 +2,9 @@ file_format: mystnb --- -# `egglog` Python - -`egglog` is a Python package that provides bindings to [the Rust library of the same name](https://github.com/egraphs-good/egglog/), -allowing you to use e-graphs in Python for optimization, symbolic computation, and analysis. - -It wraps the Rust library [`egglog`](https://github.com/egraphs-good/egglog) which -is a rewrite of the `egg` library to use [relational e-matching](https://arxiv.org/abs/2108.02290) and to add datalog features. -See the ["Better Together: Unifying Datalog and Equality Saturation"](https://arxiv.org/abs/2304.04332) paper for more details - -> We present egglog, a fixpoint reasoning system that unifies Datalog and equality saturation (EqSat). Like Datalog, it supports efficient incremental execution, cooperating analyses, and lattice-based reasoning. Like EqSat, it supports term rewriting, efficient congruence closure, and extraction of optimized terms. - -```bibtex -@misc{Shanabrook2023EgglogPython, - title = {Egglog Python: A Pythonic Library for E-graphs}, - author = {Saul Shanabrook}, - year = {2023}, - eprint = {2305.04311}, - archivePrefix = {arXiv}, - primaryClass = {cs.PL}, - doi = {10.48550/arXiv.2305.04311}, - url = {https://arxiv.org/abs/2305.04311}, - note = {Presented at EGRAPHS@PLDI 2023} -} -``` - - -```shell -pip install egglog -``` - -```{code-cell} python -from __future__ import annotations -from egglog import * - - -class Num(Expr): - def __init__(self, value: i64Like) -> None: ... - - @classmethod - def var(cls, name: StringLike) -> Num: ... - - def __add__(self, other: Num) -> Num: ... - - def __mul__(self, other: Num) -> Num: ... - -egraph = EGraph() - -expr1 = egraph.let("expr1", Num(2) * (Num.var("x") + Num(3))) -expr2 = egraph.let("expr2", Num(6) + Num(2) * Num.var("x")) - -@egraph.register -def _num_rule(a: Num, b: Num, c: Num, i: i64, j: i64): - yield rewrite(a + b).to(b + a) - yield rewrite(a * (b + c)).to((a * b) + (a * c)) - yield rewrite(Num(i) + Num(j)).to(Num(i + j)) - yield rewrite(Num(i) * Num(j)).to(Num(i * j)) - -egraph.saturate() -egraph.check(expr1 == expr2) -egraph.extract(expr1) +```{include} ../README.md +:start-after: # egglog Python +:end-before: ## License ``` ```{toctree} diff --git a/docs/reference/contributing.md b/docs/reference/contributing.md index 5ba0125c..e6420626 100644 --- a/docs/reference/contributing.md +++ b/docs/reference/contributing.md @@ -1,5 +1,37 @@ # Contributing +## Quick Reference + +For rapid development, here are the most common commands: + +```bash +# Setup +uv sync --all-extras + +# After changing Rust code in src/ +uv sync --reinstall-package egglog --all-extras + +# Testing +uv run pytest --benchmark-disable -vvv # All tests +uv run pytest -k "test_name" --benchmark-disable # Specific test + +# Code quality +uv run ruff format . # Format code +uv run ruff check --fix . # Fix linting issues +make mypy # Type checking +make stubtest # Validate type stubs + +# Documentation +make docs # Build documentation +``` + +## Project Overview + +This repository provides Python bindings for the Rust library `egglog`, enabling the use of e-graphs in Python for optimization, symbolic computation, and analysis. It is a hybrid project combining: +- **Python code** in `python/egglog/` - The main Python API and library +- **Rust code** in `src/` - PyO3-based bindings to the egglog Rust library +- **Documentation** in `docs/` - Sphinx-based documentation + ## Development This package is in active development and welcomes contributions! @@ -38,6 +70,19 @@ Anytime you change the rust code, you can run `uv sync --reinstall-package egglo If you would like to download a new version of the visualizer source, run `make clean; make`. This will download the most recent released version from the github actions artifact in the [egraph-visualizer](https://github.com/egraphs-good/egraph-visualizer) repo. It is checked in because it's a pain to get cargo to include only one git ignored file while ignoring the rest of the files that were ignored. +### Verify Your Setup + +After installation, verify everything works: + +```bash +# Quick validation (runs in ~30s) +uv run pytest -k "test_simple" --benchmark-disable +make mypy +uv run ruff check . +``` + +If these pass, you're ready to contribute! + ### Running Tests To run the tests, you can use the `pytest` command: @@ -72,6 +117,206 @@ Finally, to build the docs locally and test that they work, you can run: make docs ``` +## Repository Structure + +### Core Directories + +- `python/egglog/` - Main Python package source code +- `python/tests/` - Python test suite (pytest-based) +- `src/` - Rust source code for Python bindings (PyO3) +- `docs/` - Documentation source files (Sphinx) +- `test-data/` - Test data files +- `pyproject.toml` - Python project configuration and dependencies +- `Cargo.toml` - Rust project configuration +- `uv.lock` - Locked dependencies (managed by uv) + +### Rust Source Organization + +- `src/lib.rs` - Main library entry point +- `src/egraph.rs` - E-graph implementation +- `src/conversions.rs` - Type conversions between Python and Rust +- `src/py_object_sort.rs` - Python object handling +- `src/extract.rs` - Extraction functionality +- `src/error.rs` - Error handling +- `src/serialize.rs` - Serialization support +- `src/termdag.rs` - Term DAG operations +- `src/utils.rs` - Utility functions + +### Python Source Organization + +#### Public Interface +All public Python APIs are exported from the top-level `egglog` module. Anything that is public should be exported in `python/egglog/__init__.py` at the top level. + +#### Lower-Level Bindings +The `egglog.bindings` module provides lower-level access to the Rust implementation for advanced use cases. + +#### Core Python Files +- `python/egglog/__init__.py` - Top-level module exports, defines the public API +- `python/egglog/egraph.py` - Main EGraph class and e-graph management +- `python/egglog/egraph_state.py` - E-graph state and execution management +- `python/egglog/runtime.py` - Runtime system for expression evaluation and method definitions +- `python/egglog/builtins.py` - Built-in types (i64, f64, String, Vec, etc.) and operations +- `python/egglog/declarations.py` - Class, function, and method declaration decorators +- `python/egglog/conversion.py` - Type conversion between Python and egglog types +- `python/egglog/pretty.py` - Pretty printing for expressions and e-graph visualization +- `python/egglog/deconstruct.py` - Deconstruction of Python values into egglog expressions +- `python/egglog/thunk.py` - Lazy evaluation support +- `python/egglog/type_constraint_solver.py` - Type inference and constraint solving +- `python/egglog/config.py` - Configuration settings +- `python/egglog/ipython_magic.py` - IPython/Jupyter integration +- `python/egglog/visualizer_widget.py` - Interactive visualization widget +- `python/egglog/version_compat.py` - Python version compatibility utilities +- `python/egglog/examples/` - End-to-end samples and tutorials demonstrating the API +- `python/egglog/exp/` - Experimental Array API integrations and code generation helpers + +The compiled extension artifact `python/egglog/bindings.cpython-*.so` is generated by `uv sync` and should not be edited manually. + +## Architecture Overview + +### Python-Rust Integration + +This is a hybrid Python-Rust project that uses **PyO3** to expose Rust functionality to Python: + +1. **Rust Core**: High-performance e-graph operations in `src/` using the upstream `egglog` Rust library +2. **PyO3 Bridge**: Bindings in `src/conversions.rs` handle type conversion between Python and Rust +3. **Python API**: High-level, Pythonic interface in `python/egglog/` wraps the Rust bindings +4. **Build Process**: Maturin compiles Rust code into a Python extension module (`bindings.cpython-*.so`) + +### When Rust Rebuilds Are Needed + +**Rust changes require rebuild:** +- Any edits to files in `src/` +- Changes to `Cargo.toml` dependencies +- Run: `uv sync --reinstall-package egglog --all-extras` + +**Python-only changes (no rebuild needed):** +- Edits to `python/egglog/*.py` files +- Changes to tests in `python/tests/` +- Documentation updates + +### The Conversion Registry + +The library uses a global conversion registry to map Python types to egglog types. This registry: +- Is populated when you define classes with `@egraph.class_` or similar decorators +- Persists across e-graph instances in the same Python process +- Is automatically reset between test runs (see `python/tests/conftest.py`) + +## Code Standards + +### Python Code Style + +- **Line length**: 120 characters maximum +- **Type hints**: Use type annotations for public APIs and functions +- **Formatting**: Use Ruff for code formatting and linting +- **Testing**: Write tests using pytest in `python/tests/` +- **Docstrings**: Use clear, concise docstrings for public functions and classes + +### Ruff Configuration +The project uses Ruff for linting and formatting with specific rules: +- Allows uppercase variable names (N806, N802) +- Allows star imports (F405, F403) +- Allows `exec` and subprocess usage (S102, S307, S603) +- Allows `Any` type annotations (ANN401) +- Test files don't require full type annotations + +See `pyproject.toml` for complete Ruff configuration. + +### Type Checking +- **mypy** is used for static type checking +- Run `make mypy` to type check Python code +- Run `make stubtest` to validate type stubs against runtime behavior +- Exclusions: `__snapshots__`, `_build`, `conftest.py` + +### Naming Conventions +- Python: snake_case for functions and variables, PascalCase for classes +- Rust: Follow standard Rust conventions + +### Rust Code Standards + +- **Edition**: Rust 2024 (experimental) +- **FFI**: Uses PyO3 for Python bindings +- **Main library**: Uses egglog from git (egraphs-good/egglog, fix-fn-bug branch) + +## Testing Strategy + +### Test Organization + +Tests are located in `python/tests/` with the following categories: + +1. **Unit tests**: Test individual functions and classes +2. **Integration tests**: Test complete workflows +3. **Snapshot tests**: Use syrupy for snapshot testing of complex outputs +4. **Benchmarks**: Performance testing with pytest-benchmark and pytest-codspeed +5. **Parallel testing**: Use pytest-xdist for faster test runs +6. **Type checking**: Validate type stubs and annotations + +### Running Specific Tests + +```bash +# All tests +uv run pytest --benchmark-disable -vvv --durations=10 + +# Specific test file +uv run pytest python/tests/test_high_level.py --benchmark-disable + +# Specific test function +uv run pytest -k "test_simple" --benchmark-disable + +# Tests matching a pattern +uv run pytest -k "convert" --benchmark-disable +``` + +### Test Markers + +The project uses pytest markers to categorize tests: +- `@pytest.mark.slow` - Longer-running tests (deselect with `-m "not slow"`) + +## Common Development Patterns + +### Python API Design +- Define e-graph classes by inheriting from `egglog.Expr` +- Use `@egraph.function` decorator for functions +- Use `@egraph.method` decorator for methods +- Leverage type annotations for better IDE support + +### Working with Values +- Use `get_literal_value(expr)` or the `.value` property to get Python values from primitives +- Use pattern matching with `match`/`case` for destructuring egglog primitives +- Use `get_callable_fn(expr)` to get the underlying Python function from a callable expression +- Use `get_callable_args(expr)` to get arguments to a callable + +### Parallelism +- The underlying Rust library uses Rayon for parallelism +- Control worker thread count via `RAYON_NUM_THREADS` environment variable +- Defaults to single thread if not set + +### Rust-Python Integration +- Use PyO3's `#[pyclass]` and `#[pymethods]` macros +- Handle errors with appropriate Python exceptions +- Convert between Rust and Python types in `conversions.rs` + +## Troubleshooting + +### Rust changes not taking effect +**Problem**: Modified Rust code but changes aren't visible in Python +**Solution**: Run `uv sync --reinstall-package egglog --all-extras` to force rebuild + +### Test failures after dependency changes +**Problem**: Tests fail with import errors after updating dependencies +**Solution**: Run `uv sync --all-extras` to sync dependencies + +### Type stub mismatches +**Problem**: MyPy reports errors about bindings module +**Solution**: Run `make stubtest` to validate stubs, may need to regenerate type stubs + +### Pre-commit hooks failing +**Problem**: Pre-commit hooks reject commits +**Solution**: Run `uv run ruff format .` and `uv run ruff check --fix .` to auto-fix issues + +### Documentation build fails +**Problem**: `make docs` fails with errors +**Solution**: Ensure all extras are installed with `uv sync --all-extras`, check Sphinx warnings + ## Debugging To debug the Rust parts of this project, follow the [PyO3 debugging guide](https://pyo3.rs/main/debugging.html#debugger-specific-setup). @@ -86,23 +331,46 @@ If there is a performance sensitive piece of code, you could isolate it in a fil uv run py-spy record --format speedscope -- python -O tmp.py ``` -### Making changes +## Making Changes All changes that impact users should be documented in the `docs/changelog.md` file. Please also add tests for any new features or bug fixes. When you are ready to submit your changes, please open a pull request. The CI will run the tests and check the code style. -#### Changelog Automation +### Changelog Automation When you open a pull request, a GitHub Action automatically adds an entry to the UNRELEASED section of the changelog using your PR title and number. This ensures the changelog stays up-to-date without manual intervention. +### Contributing Guidelines + +When making changes: +1. Update or add tests in `python/tests/` for Python changes +2. Run the full test suite before committing +3. Ensure type checking passes with `make mypy` +4. Build documentation if changing public APIs +5. Follow existing code patterns and style +6. Keep changes minimal and focused +7. Ensure the automatic changelog entry in `docs/changelog.md` (added when opening the PR) accurately reflects your change and add manual notes if additional clarification is needed + ## Documentation We use the [DiΓ‘taxis framework](https://diataxis.fr/) to organize our documentation. The "explanation" section has been renamed to "Blog" since most of the content there is more like a blog post than a reference manual. It uses the [ABlog](https://ablog.readthedocs.io/en/stable/index.html#how-it-works) extension. +Documentation is built with Sphinx: +- Source files in `docs/` +- Build with `make docs` +- Output in `docs/_build/html/` +- Hosted on ReadTheDocs + +## Performance Considerations + +- The library uses Rust for performance-critical operations +- Benchmarking is done via CodSpeed for continuous performance monitoring +- Profile with release builds (`cargo build --release`) when needed + ## Governance The governance is currently informal, with Saul Shanabrook as the lead maintainer. If the project grows and there