Skip to content

Commit 2a5e90c

Browse files
tirantiran
committed
feat: add 'fromager resolve package' subcommand
The new `resole package` subcommand is designed to debug Fromager settings and resolver properties. It compares the results from Fromager's internal resolver with a simple PyPI lookup. It prints informations like resolver configuration, detected versions and versions with sdists and wheels on PyPI. ``` $ fromager --debug resolve package 'fromager~=0.68.0' 15:28:52 INFO loading settings from .../overrides/settings.yaml 15:28:52 INFO fromager: resolving requirement for 'fromager~=0.68.0' 15:28:52 INFO fromager: sdist server url: https://pypi.org/simple/ 15:28:52 INFO fromager: resolver includes sdists: True 15:28:52 INFO fromager: resolver includes wheels: False 15:28:52 INFO fromager: download url: None 15:28:52 INFO fromager: get_resolver_provider returns 'PyPIProvider' 15:28:52 INFO fromager: found 2 Fromager candidates with 2 unique versions 15:28:52 INFO fromager: Fromager versions: 0.68.0, 0.68.1 15:28:52 INFO fromager: found 2 versions on PyPI 15:28:52 INFO fromager: PyPI versions: 0.68.0, 0.68.1 15:28:52 INFO fromager: - with sdists: 0.68.0, 0.68.1 15:28:52 INFO fromager: - with wheels: 0.68.0, 0.68.1 ``` Signed-off-by: Christian Heimes <cheimes@redhat.com>
1 parent 6f9a334 commit 2a5e90c

4 files changed

Lines changed: 204 additions & 0 deletions

File tree

‎pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ list-overrides = "fromager.commands.list_overrides:list_overrides"
119119
list-versions = "fromager.commands.list_versions:list_versions"
120120
migrate-config = "fromager.commands.migrate_config:migrate_config"
121121
minimize = "fromager.commands.minimize:minimize"
122+
resolve = "fromager.commands.resolve:resolve"
122123
stats = "fromager.commands.stats:stats"
123124
step = "fromager.commands.step:step"
124125
canonicalize = "fromager.commands.canonicalize:canonicalize"

‎src/fromager/commands/resolve.py‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
import logging
2+
import sys
3+
import typing
4+
5+
import click
6+
import pypi_simple
7+
from packaging.requirements import Requirement
8+
from packaging.version import Version
9+
from resolvelib.resolvers import ResolverException
10+
11+
from fromager import context, log, overrides, request_session, resolver
12+
13+
logger = logging.getLogger(__name__)
14+
15+
16+
@click.group()
17+
def resolve() -> None:
18+
"Commands for resolving packages and dependencies"
19+
pass
20+
21+
22+
def _versions_string(versions: typing.Iterable[Version]) -> str:
23+
return ", ".join(str(version) for version in sorted(versions))
24+
25+
26+
@resolve.command()
27+
@click.argument(
28+
"package",
29+
)
30+
@click.pass_obj
31+
def package(
32+
wkctx: context.WorkContext,
33+
package: str,
34+
) -> None:
35+
"""Resolve a package with Fromager's resolver and PyPI
36+
37+
The package resolver subcommand is a debug tool. It shows information
38+
about the package's resolver configuration and resolves a package in two
39+
way. First, it resolves with Fromager's resolver. Second, it performs a
40+
simple query against PyPI and compares the results.
41+
42+
Usage::
43+
44+
$ fromager --variant cpu package resolve 'fromager>=0.70'
45+
"""
46+
47+
req = Requirement(package)
48+
# wkctx.constraints.add_constraint(package)
49+
50+
with log.req_ctxvar_context(req):
51+
logger.info(
52+
"resolving requirement for %r (variant: %s)", package, wkctx.variant
53+
)
54+
55+
global_constraint = wkctx.constraints.get_constraint(req.name)
56+
if global_constraint is not None:
57+
logger.warning("Package has a global constraint %s.", global_constraint)
58+
else:
59+
logger.info("Package has not global constraint.")
60+
61+
# print package build info settings (overrides)
62+
pbi = wkctx.package_build_info(req)
63+
override_sdist_server_url = pbi.resolver_sdist_server_url(
64+
"https://pypi.org/simple/"
65+
)
66+
include_sdists = pbi.resolver_include_sdists
67+
include_wheels = pbi.resolver_include_wheels
68+
69+
if pbi.has_customizations:
70+
logger.warning(
71+
"Package has customizations (config override, plugin, patches)."
72+
)
73+
else:
74+
logger.info("Package uses standard settings.")
75+
logger.info("- sdist server url: %s", override_sdist_server_url)
76+
logger.info("- resolver includes sdists: %s", pbi.resolver_include_sdists)
77+
logger.info("- resolver includes wheels: %s", pbi.resolver_include_wheels)
78+
logger.info("- wheel server url: %s", pbi.wheel_server_url)
79+
logger.info(
80+
"- download url: %s", pbi.download_source_url(resolve_template=False)
81+
)
82+
logger.info("- prebuilt wheel: %s", pbi.pre_built)
83+
84+
# resolve package with Fromager's resolver and settings.
85+
provider = overrides.find_and_invoke(
86+
req.name,
87+
"get_resolver_provider",
88+
resolver.default_resolver_provider,
89+
ctx=wkctx,
90+
req=req,
91+
include_sdists=include_sdists,
92+
include_wheels=include_wheels,
93+
sdist_server_url=override_sdist_server_url,
94+
)
95+
96+
logger.info(
97+
"get_resolver_provider returns provider %r", type(provider).__name__
98+
)
99+
100+
fromager_versions: set[Version]
101+
try:
102+
candidates = list(
103+
provider.find_matches(
104+
identifier=req.name,
105+
requirements={req.name: [req]},
106+
incompatibilities={req.name: []},
107+
)
108+
)
109+
except ResolverException as e:
110+
logger.error(str(e))
111+
fromager_versions = set()
112+
else:
113+
fromager_versions = set(candidate.version for candidate in candidates)
114+
logger.info(
115+
"found %i Fromager candidates with %i unique versions for req: '%s', constraint: '%s'",
116+
len(candidates),
117+
len(fromager_versions),
118+
req,
119+
global_constraint,
120+
)
121+
logger.info("Fromager versions: %s", _versions_string(fromager_versions))
122+
123+
# resolve package from PyPI
124+
pypi_client = pypi_simple.PyPISimple(
125+
accept=pypi_simple.ACCEPT_JSON_ONLY,
126+
session=request_session.session,
127+
)
128+
try:
129+
pypi_package = pypi_client.get_project_page(req.name)
130+
except Exception as e:
131+
logger.debug(
132+
"failed to fetch package index from pypi.org: %s",
133+
e,
134+
)
135+
else:
136+
pypi_versions_wheels: set[Version] = set()
137+
pypi_versions_sdists: set[Version] = set()
138+
for pkg in pypi_package.packages:
139+
if pkg.package_type not in {"sdist", "wheel"} or not pkg.version:
140+
continue
141+
version = Version(pkg.version)
142+
if pkg.is_yanked:
143+
logger.debug("%s is yanked")
144+
continue
145+
if req.specifier and version not in req.specifier:
146+
logger.debug(
147+
"%s is not in requirment specifier %s", version, req.specifier
148+
)
149+
continue
150+
if (
151+
global_constraint is not None
152+
and version not in global_constraint.specifier
153+
):
154+
logger.debug(
155+
"%s is excluded by global constraint %s",
156+
version,
157+
global_constraint,
158+
)
159+
continue
160+
if pkg.package_type == "wheel":
161+
pypi_versions_wheels.add(version)
162+
else:
163+
pypi_versions_sdists.add(version)
164+
165+
pypi_versions: set[Version] = pypi_versions_sdists | pypi_versions_wheels
166+
167+
logger.info(
168+
"found %i versions on PyPI for req: '%s', constraint: '%s'",
169+
len(pypi_versions),
170+
req,
171+
global_constraint,
172+
)
173+
logger.info("PyPI versions: %s", _versions_string(pypi_versions))
174+
logger.info("- with sdists: %s", _versions_string(pypi_versions_sdists))
175+
logger.info("- with wheels: %s", _versions_string(pypi_versions_wheels))
176+
177+
diff = fromager_versions.difference(pypi_versions)
178+
if diff:
179+
logger.warning("missing from PyPI: %s", _versions_string(diff))
180+
diff = pypi_versions.difference(fromager_versions)
181+
if diff:
182+
logger.warning("missing from Fromager: %s", _versions_string(diff))
183+
184+
if not fromager_versions:
185+
logger.error("Fromager lookup failed")
186+
sys.exit(2)

‎tests/test_cli.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ def test_fromager_version(cli_runner: CliRunner) -> None:
5959
"list-versions",
6060
"migrate-config",
6161
"minimize",
62+
"resolve",
6263
"stats",
6364
"step",
6465
"wheel-server",

‎tests/test_resolver.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55
import pytest
66
import requests_mock
77
import resolvelib
8+
from click.testing import CliRunner
89
from packaging.requirements import Requirement
910
from packaging.version import Version
1011

1112
from fromager import constraints, resolver
13+
from fromager.__main__ import main as fromager
1214

1315
_hydra_core_simple_response = """
1416
<!DOCTYPE html>
@@ -1109,3 +1111,17 @@ def custom_resolver_provider(
11091111
assert "pypi.org" not in error_message.lower(), (
11101112
f"Error message incorrectly mentions PyPI when using GitHub resolver: {error_message}"
11111113
)
1114+
1115+
1116+
def test_cli_package_resolver(
1117+
cli_runner: CliRunner,
1118+
caplog: pytest.LogCaptureFixture,
1119+
pypi_hydra_resolver: typing.Any,
1120+
) -> None:
1121+
result = cli_runner.invoke(fromager, ["resolve", "package", "hydra-core"])
1122+
assert result.exit_code == 0
1123+
assert "Fromager versions: 1.2.2, 1.3.2" in caplog.messages
1124+
assert "PyPI versions: 1.2.2, 1.3.1+local, 1.3.2, 2.0.0a1" in caplog.messages
1125+
assert "- with sdists: 1.2.2, 1.3.2" in caplog.messages
1126+
assert "- with wheels: 1.2.2, 1.3.1+local, 1.3.2, 2.0.0a1" in caplog.messages
1127+
assert "missing from Fromager: 1.3.1+local, 2.0.0a1" in caplog.messages

0 commit comments

Comments
 (0)