Skip to content

Commit 02fbce3

Browse files
mergify[bot]mergify[bot]
authored
Merge pull request #508 from shubhbapna/use-zipfile
use zipfile library instead of wheel command to unpack a wheel
2 parents d90bc08 + 67e414b commit 02fbce3

3 files changed

Lines changed: 59 additions & 59 deletions

File tree

‎e2e/test_bootstrap_cache.sh‎

Lines changed: 37 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ fromager \
2020
start_local_wheel_server
2121
rm -rf "$OUTDIR/sdists-repo"
2222
rm -rf "$OUTDIR/work-dir"
23+
rm "$OUTDIR/bootstrap.log"
2324

2425
# run fromager with the cache wheel server pointing to the local wheel server
2526
fromager \
@@ -32,6 +33,28 @@ fromager \
3233
--no-cleanup \
3334
bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
3435

36+
EXPECTED_LOG_MESSAGES=(
37+
"loading build sdist dependencies from build-sdist-requirements.txt"
38+
"loading build backend dependencies from build-backend-requirements.txt"
39+
"loading build system dependencies from build-system-requirements.txt"
40+
)
41+
42+
for pattern in "${EXPECTED_LOG_MESSAGES[@]}"; do
43+
echo $pattern
44+
if ! grep -q "stevedore: $pattern" "$OUTDIR/bootstrap.log"; then
45+
echo "FAIL: Did not find log message stevedore: $pattern in $OUTDIR/bootstrap.log" 1>&2
46+
pass=false
47+
fi
48+
if ! grep -q "pbr: $pattern" "$OUTDIR/bootstrap.log"; then
49+
echo "FAIL: Did not find log message pbr: $pattern in $OUTDIR/bootstrap.log" 1>&2
50+
pass=false
51+
fi
52+
if ! grep -q "setuptools: $pattern" "$OUTDIR/bootstrap.log"; then
53+
echo "FAIL: Did not find log message setuptools: $pattern in $OUTDIR/bootstrap.log" 1>&2
54+
pass=false
55+
fi
56+
done
57+
3558
EXPECTED_FILES="
3659
$OUTDIR/wheels-repo/downloads/setuptools-*.whl
3760
$OUTDIR/wheels-repo/downloads/pbr-*.whl
@@ -41,9 +64,9 @@ $OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz
4164
$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
4265
$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
4366
44-
$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info/fromager-*.txt
45-
$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info/fromager-*.txt
46-
$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info/fromager-*.txt
67+
$OUTDIR/work-dir/pbr-*/*-requirements.txt
68+
$OUTDIR/work-dir/setuptools-*/*-requirements.txt
69+
$OUTDIR/work-dir/stevedore-*/*-requirements.txt
4770
4871
$OUTDIR/work-dir/build-order.json
4972
$OUTDIR/work-dir/constraints.txt
@@ -61,6 +84,7 @@ $pass
6184
rm -rf "$OUTDIR/sdists-repo"
6285
rm -rf "$OUTDIR/work-dir"
6386
rm -rf "$OUTDIR/wheels-repo"
87+
rm "$OUTDIR/bootstrap.log"
6488

6589
# run fromager with the cache wheel server pointing to the pypi server
6690
fromager \
@@ -95,30 +119,18 @@ done
95119

96120
$pass
97121

98-
NOT_EXPECTED_FILES="
99-
$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info/fromager-*.txt
100-
$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info/fromager-*.txt
101-
$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info/fromager-*.txt
102-
"
103-
104-
for pattern in $NOT_EXPECTED_FILES; do
105-
if [ -f "${pattern}" ]; then
106-
echo "Found $pattern" 1>&2
122+
for pattern in "${EXPECTED_LOG_MESSAGES[@]}"; do
123+
echo $pattern
124+
if grep -q "stevedore: $pattern" "$OUTDIR/bootstrap.log"; then
125+
echo "FAIL: found log message stevedore: $pattern in $OUTDIR/bootstrap.log" 1>&2
107126
pass=false
108127
fi
109-
done
110-
111-
$pass
112-
113-
EXPECTED_DIR="
114-
$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info
115-
$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info
116-
$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info
117-
"
118-
119-
for pattern in $EXPECTED_DIR; do
120-
if [ -d "${pattern}" ]; then
121-
echo "Did not find $pattern" 1>&2
128+
if grep -q "pbr: $pattern" "$OUTDIR/bootstrap.log"; then
129+
echo "FAIL: found log message pbr: $pattern in $OUTDIR/bootstrap.log" 1>&2
130+
pass=false
131+
fi
132+
if grep -q "setuptools: $pattern" "$OUTDIR/bootstrap.log"; then
133+
echo "FAIL: found log message setuptools: $pattern in $OUTDIR/bootstrap.log" 1>&2
122134
pass=false
123135
fi
124136
done

‎src/fromager/bootstrapper.py‎

Lines changed: 19 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import pathlib
77
import shutil
88
import typing
9+
import zipfile
910
from urllib.parse import urlparse
1011

1112
from packaging.requirements import Requirement
@@ -15,7 +16,6 @@
1516
from . import (
1617
build_environment,
1718
dependencies,
18-
external_commands,
1919
finders,
2020
progress,
2121
resolver,
@@ -340,34 +340,26 @@ def _download_wheel_from_cache(
340340
server.update_wheel_mirror(self.ctx)
341341
logger.info(f"{req.name}: found built wheel on cache server")
342342
unpack_dir = self._create_unpack_dir(req, resolved_version)
343-
# unpack the wheel
344-
external_commands.run(
345-
["wheel", "unpack", str(cached_wheel), "--dest", unpack_dir],
346-
cwd=unpack_dir,
347-
)
348-
349343
dist_filename = f"{dist_name}-{dist_version}"
350-
metadata_dir = unpack_dir / dist_filename / f"{dist_filename}.dist-info"
351-
352-
# prepare files cached by fromager by placing them in the parent directory of the return path
344+
metadata_dir = pathlib.Path(f"{dist_filename}.dist-info")
353345
try:
354-
shutil.copy(
355-
metadata_dir
356-
/ f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{dependencies.BUILD_SYSTEM_REQ_FILE_NAME}",
357-
unpack_dir / dependencies.BUILD_SYSTEM_REQ_FILE_NAME,
358-
)
359-
shutil.copy(
360-
metadata_dir
361-
/ f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{dependencies.BUILD_BACKEND_REQ_FILE_NAME}",
362-
unpack_dir / dependencies.BUILD_BACKEND_REQ_FILE_NAME,
363-
)
364-
shutil.copy(
365-
metadata_dir
366-
/ f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{dependencies.BUILD_SDIST_REQ_FILE_NAME}",
367-
unpack_dir / dependencies.BUILD_SDIST_REQ_FILE_NAME,
368-
)
346+
archive = zipfile.ZipFile(cached_wheel)
347+
for filename in [
348+
dependencies.BUILD_BACKEND_REQ_FILE_NAME,
349+
dependencies.BUILD_SDIST_REQ_FILE_NAME,
350+
dependencies.BUILD_SYSTEM_REQ_FILE_NAME,
351+
]:
352+
zipinfo = archive.getinfo(
353+
str(
354+
metadata_dir
355+
/ f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{filename}"
356+
)
357+
)
358+
zipinfo.filename = filename
359+
archive.extract(zipinfo, unpack_dir)
360+
369361
logger.info(f"{req.name}: extracted build requirements from wheel")
370-
return unpack_dir / dist_filename, cached_wheel
362+
return unpack_dir / metadata_dir, cached_wheel
371363
except Exception:
372364
# implies that the wheel server hosted non-fromager built wheels
373365
logger.info(
@@ -542,7 +534,7 @@ def _cleanup(
542534

543535
# Cleanup the source tree and build environment, leaving any other
544536
# artifacts that were created.
545-
if sdist_root_dir:
537+
if sdist_root_dir and sdist_root_dir.exists():
546538
logger.debug(f"{req.name}: cleaning up source tree {sdist_root_dir}")
547539
shutil.rmtree(sdist_root_dir)
548540
logger.debug(f"{req.name}: cleaned up source tree {sdist_root_dir}")

‎src/fromager/wheels.py‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -151,14 +151,10 @@ def add_extra_metadata_to_wheels(
151151
data_to_add = {}
152152

153153
with tempfile.TemporaryDirectory() as dir_name:
154-
cmd = ["wheel", "unpack", str(wheel_file), "--dest", dir_name]
155-
external_commands.run(
156-
cmd,
157-
cwd=dir_name,
158-
network_isolation=ctx.network_isolation,
159-
)
160-
161154
wheel_root_dir = pathlib.Path(dir_name) / dist_filename
155+
wheel_root_dir.mkdir()
156+
zipfile.ZipFile(str(wheel_file)).extractall(str(wheel_root_dir))
157+
162158
dist_info_dir = wheel_root_dir / f"{dist_filename}.dist-info"
163159
if not dist_info_dir.is_dir():
164160
raise ValueError(

0 commit comments

Comments
 (0)