Skip to content

Commit 7f04e12

Browse files
dhellmanndhellmann
committed
let bootstrapper reuse a wheel on the local filesystem
Update the bootstrapper to reuse wheels already present on the local filesystem in the work directory instead of only going to the cache server. This ensures that someone working on multiple related wheels can reuse one expensive one when building the next. Update the test for caching in the bootstrap logic to focus on the cases of an existing wheel on the filesystem and a wheel available from any cache server. This removes the separate test for using a local cache server versus using a remote one, since there should effectively be no difference in those cases.
1 parent ca23285 commit 7f04e12

3 files changed

Lines changed: 199 additions & 68 deletions

File tree

‎e2e/test_bootstrap_build_tags.sh‎

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ source "$SCRIPTDIR/common.sh"
88

99
# bootstrap stevedore with 1 change log
1010
fromager \
11-
--log-file="$OUTDIR/bootstrap.log" \
11+
--log-file="$OUTDIR/bootstrap1.log" \
1212
--error-log-file="$OUTDIR/fromager-errors.log" \
1313
--sdists-repo="$OUTDIR/sdists-repo" \
1414
--wheels-repo="$OUTDIR/wheels-repo" \
@@ -20,8 +20,6 @@ EXPECTED_FILES="
2020
$OUTDIR/wheels-repo/downloads/stevedore-5.2.0-1*.whl
2121
"
2222

23-
start_local_wheel_server
24-
2523
pass=true
2624
for pattern in $EXPECTED_FILES; do
2725
if [ ! -f "${pattern}" ]; then
@@ -32,26 +30,64 @@ done
3230

3331
$pass
3432

33+
# bootstrap stevedore again with 1 change log without a file in the downloads
34+
# cache but with a file in the cache package server. Should not result in a
35+
# build because the file exists in the cache server directory.
36+
37+
find "$OUTDIR/wheels-repo"
38+
# replace the symlink in the index dir with an actual file so it can be
39+
# downloaded, and remove the wheels from the disk cache so we know we get the
40+
# file from the web server
41+
rm -f $OUTDIR/wheels-repo/simple/stevedore/*.whl
42+
mv $OUTDIR/wheels-repo/downloads/stevedore*.whl $OUTDIR/wheels-repo/simple/stevedore/
43+
rm -f $OUTDIR/wheels-repo/build/*.whl
44+
rm -f $OUTDIR/wheels-repo/downloads/*.whl
45+
find "$OUTDIR/wheels-repo"
46+
start_local_wheel_server
47+
48+
LOGFILE="$OUTDIR/bootstrap2.log"
49+
fromager \
50+
--log-file="$LOGFILE" \
51+
--error-log-file="$OUTDIR/fromager-errors.log" \
52+
--sdists-repo="$OUTDIR/sdists-repo" \
53+
--wheels-repo="$OUTDIR/wheels-repo" \
54+
--work-dir="$OUTDIR/work-dir" \
55+
--settings-file="$SCRIPTDIR/bootstrap_settings.yaml" \
56+
bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
57+
58+
if ! grep -q "stevedore: found built wheel on cache server" "$LOGFILE"; then
59+
echo "FAIL: Did not find log message found built wheel on cache server in $LOGFILE" 1>&2
60+
pass=false
61+
fi
62+
63+
$pass
64+
3565
# bootstrap stevedore again with 1 change log. Should not result in a build
66+
# because the file exists in the cache server directory.
67+
68+
find "$OUTDIR/wheels-repo"
69+
70+
LOGFILE="$OUTDIR/bootstrap3.log"
3671
fromager \
37-
--log-file="$OUTDIR/bootstrap.log" \
72+
--log-file="$LOGFILE" \
3873
--error-log-file="$OUTDIR/fromager-errors.log" \
3974
--sdists-repo="$OUTDIR/sdists-repo" \
4075
--wheels-repo="$OUTDIR/wheels-repo" \
4176
--work-dir="$OUTDIR/work-dir" \
4277
--settings-file="$SCRIPTDIR/bootstrap_settings.yaml" \
4378
bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
4479

45-
if ! grep -q "stevedore: found built wheel on cache server" "$OUTDIR/bootstrap.log"; then
46-
echo "FAIL: Did not find log message found built wheel on cache server in $OUTDIR/bootstrap.log" 1>&2
80+
if ! grep -q "stevedore: found existing wheel " "$LOGFILE"; then
81+
echo "FAIL: Did not find log message found existing wheel in $LOGFILE" 1>&2
4782
pass=false
4883
fi
4984

5085
$pass
5186

5287
# bootstrap stevedore with 2 changelog. should result in a build instead of being skipped
88+
LOGFILE="$OUTDIR/bootstrap4.log"
5389
fromager \
54-
--log-file="$OUTDIR/bootstrap_build_tags.log" \
90+
--log-file="$LOGFILE" \
5591
--error-log-file="$OUTDIR/fromager-errors.log" \
5692
--sdists-repo="$OUTDIR/sdists-repo" \
5793
--wheels-repo="$OUTDIR/wheels-repo" \
@@ -72,8 +108,8 @@ for pattern in $EXPECTED_FILES; do
72108
fi
73109
done
74110

75-
if ! grep -q "added extra metadata and build tag" "$OUTDIR/bootstrap_build_tags.log"; then
76-
echo "Did not find message indicating builds would be skipped" 1>&2
111+
if ! grep -q "added extra metadata and build tag" "$LOGFILE"; then
112+
echo "Did not find message indicating builds would be skipped in $LOGFILE" 1>&2
77113
pass=false
78114
fi
79115

‎e2e/test_bootstrap_cache.sh‎

Lines changed: 40 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -7,80 +7,86 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
77
source "$SCRIPTDIR/common.sh"
88
pass=true
99

10+
DIST=setuptools
11+
VER=78.1.0
12+
13+
################################################################################
1014
# run fromager once to build wheels that can be used by a local wheel server
15+
1116
fromager \
17+
--debug \
1218
--log-file="$OUTDIR/bootstrap.log" \
1319
--error-log-file="$OUTDIR/fromager-errors.log" \
1420
--sdists-repo="$OUTDIR/sdists-repo" \
1521
--wheels-repo="$OUTDIR/wheels-repo" \
1622
--work-dir="$OUTDIR/work-dir" \
1723
--settings-dir="$SCRIPTDIR/changelog_settings" \
18-
bootstrap 'setuptools'
24+
bootstrap "$DIST==$VER"
1925

20-
start_local_wheel_server
21-
rm -rf "$OUTDIR/sdists-repo"
26+
################################################################################
27+
# run fromager again to verify that we pick up the existing wheels in the output directory
28+
29+
# Remove build work-dir and wheel server dir, but not the sdist or wheel repos
2230
rm -rf "$OUTDIR/work-dir"
31+
rm -rf "$OUTDIR/wheels-repo/simple"
2332
rm "$OUTDIR/bootstrap.log"
2433

25-
26-
# run fromager with the cache wheel server pointing to the local wheel server
2734
fromager \
35+
--debug \
2836
--log-file="$OUTDIR/bootstrap.log" \
2937
--error-log-file="$OUTDIR/fromager-errors.log" \
3038
--sdists-repo="$OUTDIR/sdists-repo" \
3139
--wheels-repo="$OUTDIR/wheels-repo" \
3240
--work-dir="$OUTDIR/work-dir" \
3341
--settings-dir="$SCRIPTDIR/changelog_settings" \
34-
--no-cleanup \
35-
bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'setuptools'
42+
bootstrap --cache-wheel-server-url="https://pypi.org/simple" "$DIST==$VER"
3643

3744
EXPECTED_LOG_MESSAGES=(
38-
"setuptools: loading build sdist dependencies from build-sdist-requirements.txt"
39-
"setuptools: loading build backend dependencies from build-backend-requirements.txt"
40-
"setuptools: loading build system dependencies from build-system-requirements.txt"
45+
"$DIST: looking for existing wheel for version $VER with build tag () in"
46+
"$DIST: found existing wheel"
4147
)
42-
4348
for pattern in "${EXPECTED_LOG_MESSAGES[@]}"; do
44-
echo $pattern
4549
if ! grep -q "$pattern" "$OUTDIR/bootstrap.log"; then
4650
echo "FAIL: Did not find log message $pattern in $OUTDIR/bootstrap.log" 1>&2
4751
pass=false
4852
fi
4953
done
54+
$pass
5055

51-
EXPECTED_FILES="
52-
$OUTDIR/wheels-repo/downloads/setuptools-*.whl
53-
54-
$OUTDIR/work-dir/setuptools-*/*-requirements.txt
55-
56-
$OUTDIR/work-dir/build-order.json
57-
$OUTDIR/work-dir/constraints.txt
58-
"
56+
UNEXPECTED_LOG_MESSAGES=(
57+
"$DIST: checking if wheel was already uploaded to https://pypi.org/simple"
58+
)
5959

60-
for pattern in $EXPECTED_FILES; do
61-
if [ ! -f "${pattern}" ]; then
62-
echo "Did not find $pattern" 1>&2
60+
for pattern in "${UNEXPECTED_LOG_MESSAGES[@]}"; do
61+
if grep -q "$pattern" "$OUTDIR/bootstrap.log"; then
62+
echo "FAIL: Found log message $pattern in $OUTDIR/bootstrap.log" 1>&2
6363
pass=false
6464
fi
6565
done
66-
6766
$pass
6867

68+
################################################################################
69+
70+
# run fromager with the cache wheel server pointing to the pypi server and
71+
# verify we can pick it up from there
72+
73+
start_local_wheel_server
6974
rm -rf "$OUTDIR/sdists-repo"
7075
rm -rf "$OUTDIR/work-dir"
71-
rm -rf "$OUTDIR/wheels-repo"
76+
rm -rf "$OUTDIR/wheels-repo/build"
77+
rm -rf "$OUTDIR/wheels-repo/download"
7278
rm "$OUTDIR/bootstrap.log"
7379

74-
# run fromager with the cache wheel server pointing to the pypi server
7580
fromager \
81+
--debug \
7682
--log-file="$OUTDIR/bootstrap.log" \
7783
--error-log-file="$OUTDIR/fromager-errors.log" \
7884
--sdists-repo="$OUTDIR/sdists-repo" \
7985
--wheels-repo="$OUTDIR/wheels-repo" \
8086
--work-dir="$OUTDIR/work-dir" \
8187
--settings-dir="$SCRIPTDIR/changelog_settings" \
8288
--no-cleanup \
83-
bootstrap --cache-wheel-server-url="https://pypi.org/simple" 'setuptools'
89+
bootstrap --cache-wheel-server-url="https://pypi.org/simple" "$DIST==$VER"
8490

8591
EXPECTED_FILES="
8692
$OUTDIR/wheels-repo/downloads/setuptools-*.whl
@@ -98,7 +104,13 @@ done
98104

99105
$pass
100106

101-
for pattern in "${EXPECTED_LOG_MESSAGES[@]}"; do
107+
UNEXPECTED_LOG_MESSAGES=(
108+
"$DIST: loading build sdist dependencies from build-sdist-requirements.txt"
109+
"$DIST: loading build backend dependencies from build-backend-requirements.txt"
110+
"$DIST: loading build system dependencies from build-system-requirements.txt"
111+
)
112+
113+
for pattern in "${UNEXPECTED_LOG_MESSAGES[@]}"; do
102114
echo $pattern
103115
if grep -q "$pattern" "$OUTDIR/bootstrap.log"; then
104116
echo "FAIL: found log message $pattern in $OUTDIR/bootstrap.log" 1>&2

0 commit comments

Comments
 (0)