From f673496d50550b2113bf934199439198c70af7ab Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 27 Feb 2026 11:32:05 +0100 Subject: [PATCH] [proposal] New resolver configuration Add design proposal for new resolver configuration. See: #937 Signed-off-by: Christian Heimes --- docs/conf.py | 1 + docs/index.rst | 1 + docs/proposals/index.rst | 7 ++ docs/proposals/new-resolver-config.md | 144 ++++++++++++++++++++++++++ docs/spelling_wordlist.txt | 66 ++++++++++++ pyproject.toml | 1 + 6 files changed, 220 insertions(+) create mode 100644 docs/proposals/index.rst create mode 100644 docs/proposals/new-resolver-config.md create mode 100644 docs/spelling_wordlist.txt diff --git a/docs/conf.py b/docs/conf.py index 7343676b..58034423 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,6 +28,7 @@ "sphinxcontrib.autodoc_pydantic", "sphinx.ext.autodoc", "sphinx.ext.intersphinx", + "sphinxcontrib.spelling", ] # Enable MyST extensions to support reStructuredText directives in Markdown diff --git a/docs/index.rst b/docs/index.rst index f0bd4e5e..71597967 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,6 +39,7 @@ those special cases directly into fromager. cli.rst glossary.rst develop.md + proposals/index.rst What's with the name? --------------------- diff --git a/docs/proposals/index.rst b/docs/proposals/index.rst new file mode 100644 index 00000000..ad4bc012 --- /dev/null +++ b/docs/proposals/index.rst @@ -0,0 +1,7 @@ +Fromager Enhancement Proposals +============================== + +.. toctree:: + :maxdepth: 1 + + new-resolver-config diff --git a/docs/proposals/new-resolver-config.md b/docs/proposals/new-resolver-config.md new file mode 100644 index 00000000..fc2de1e2 --- /dev/null +++ b/docs/proposals/new-resolver-config.md @@ -0,0 +1,144 @@ +# New resolver and download configuration + +- Author: Christian Heimes +- Created: 2026-02-24 +- Status: Open + +## What + +This enhancement document proposal a new approach to configure the package +resolver and source / sdist downloader. The new settings are covering a +wider range of use cases. Common patterns like building a package from a +git checkout will no longer need custom Python plugins. + +## Why + +In downstream, we are encountering an increasing amount of packages that do +not build from sdists on PyPI. Either package maintainers are not uploading +source distributions to PyPI or sdists have issues. In some cases, packages +use a midstream fork that is not on PyPI. The sources need to be build from +git. + +Because Fromager <= 0.76 does not have declarative settings for GitHub/GitLab +resolver or cloning git repositories, we have to write custom Python plugins. +The plugins are a maintenance burden. + +## Goals + +- support common use cases with package settings instead of custom plugin code +- cover most common resolver scenarios: + - resolve package on PyPI (sdist, wheel, or both) + - resolve package on GitHub or GitLab with custom tag matcher +- cover common sdist download and build scenarios: + - sdist from PyPI + - prebuilt wheel from PyPI + - download tarball from URL + - clone git repository + - download an artifact from GitHub / GitLab release or tag + - build sdist with PEP 517 hook or plain tarball +- support per-variant setting, e.g. one variant uses prebuilt wheel while the + rest uses sdist. +- gradual migration path from old system to new configuration + +## Non-goals + +- The new system will not cover all use cases. Some specific use cases will + still require custom code. +- Retrieval of additional sources is out of scope, e.g. a package `egg` that + needs `libegg-{version}.tar.gz`. + +## How + +The new system will use a new top-level configuration key `source`. The old +`download_source` and `resolver_dist` settings will stay supported for a +while. Eventually the old options will be deprecated and removed. + +The resolver and source downloader can be configuration for all variants of +a package as well as re-defined for each variant. A package can be configured +as prebuilt for all variants or a variant can have a different resolver and +sources than other. + +Each use case is handled a provider profile. The profile name acts as a tag +([discriminated union](https://docs.pydantic.dev/latest/concepts/unions/#discriminated-unions)). +Each use case has a well-defined set of mandatory and optional arguments. + +Example: + +```yaml +source: + # `pypi-sdist` is the default provider + provider: pypi-sdist +variants: + egg: + source: + # resolve and download prebuilt wheel + provider: pypi-prebuilt + index_url: https://custom-index.test/simple + spam: + source: + # resolve on GitLab, clone, build an sdist with PEP 517 hook + provider: gitlab-git + url: https://gitlab.test/spam/spam + matcher_factory: package_plugins.matchers:midstream_matcher_factory + retrieve_method: git+https + build_sdist: pep517 + viking: + source: + # resolve on PyPI, git clone, and build as tarball + provider: pypi-git + clone_url: https://github.test/viking/viking.git + tag: 'v{version}' + build_sdist: tarball + camelot: + source: + # Camelot is a silly place, let's not go there. + provider: not-available +``` + +### Profiles + +- The `pypi-sdist` profile resolve versions on PyPI or PyPI-compatible index. + It only takes sdists into account and downloads the sdist from the index. + The profile is equivalent to the current default settings with + `include_sdists: true` and `include_wheels: false`. + +- The `pypi-prebuilt` profile resolve versions of platform-specific wheels + on PyPI and downloads the pre-built wheel. The profile is equivalent to + `include_sdists: false`, `include_wheels: true`, and variant setting + `pre_build: true`. + +- The `pypi-download` resolve versions of any package on PyPI and downloads + a tarball from an external URL (with `{version}` variable in download URL). + It takes any sdist and any wheel into account. The profile is equivalent + with `include_sdists: true`, `include_wheels: true`, `ignore_platform: true`, + and a `download_source.url`. + +- The `pypi-git` is similar to the `pypi-download` profile. Instead of + downloading a tarball, it clones a git repository at a specific tag. + +- The `gitlab-git` and `github-git` profiles use the `GitLabTagProvider` or + `GitHubTagProvider` to resolve versions. The profiles git clone a project + over `https` or `ssh` protocol. + +- The `gitlab-download` and `github-download` are similar to `gitlab-git` and + `github-git` profiles. Instead of cloning a git repository, they download + a release artifact from the release or tag. + +- The `not-available` profile raises an error. It can be used to block a + package and only enable it for a single variant. + +Like pip's VCS feature, all git clone operations automatically retrieve all +submodules recursively. + +### Deprecations + +- `download_source.url` is handled by `pypi-download` profile or + `release_artifact` parameter of `github` or `gitlab` provider +- `download_source.destination_filename` is not needed. All sdists use + standard `{dist_name}-{version}.tar.gz` file name +- `resolver_dist.sdist_server_url` is replaced by `index_url` parameter. + All `pypi-*` profile support a custom index. +- `git_options.submodules` is not needed. Like pip, Fromager will always + clone all submodules. +- variant settings `wheel_server_url` and `pre_build` are replaced by + `pypi-prebuilt` profile diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt new file mode 100644 index 00000000..ebce9fcf --- /dev/null +++ b/docs/spelling_wordlist.txt @@ -0,0 +1,66 @@ +args +backend +backend +backends +canonicalize +changelog +changelogs +cheeseshop +codebase +config +containerfile +cpu +customizations +cython +deprecations +downloader +env +filesystem +fromager +frontend +frontends +graphviz +iterable +json +linter +localhost +matcher +mypy +namespace +numpy +openssl +platlib +podman +pre +prebuilt +purelib +py +pydantic +pypi +pyproject +repo +sdist +sdists +setuptools +stderr +stdin +stdout +subcommands +subdirectory +submodule +submodules +subprocesses +toml +toplevel +traceback +tracebacks +txt +unshare +url +urls +vendored +vendoring +versionless +virtualenv +whitespace +yaml diff --git a/pyproject.toml b/pyproject.toml index 2356d2d4..2996cfcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ docs = [ "myst-parser", "sphinx-rtd-theme", "autodoc-pydantic", + "sphinxcontrib.spelling", ] [project.urls]