Skip to content

Commit 0a1de85

Browse files
mergify[bot]mergify[bot]
authored
Merge pull request #586 from dhellmann/bootstrap-check-local-dir-before-cache-server
Bootstrap check local dir before cache server
2 parents be34057 + ed30a0f commit 0a1de85

4 files changed

Lines changed: 202 additions & 94 deletions

File tree

‎e2e/common.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ set -e
55
set -u
66
set -o pipefail
77

8+
export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
9+
810
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
911
OUTDIR="$(dirname "$SCRIPTDIR")/e2e-output"
1012

‎e2e/test_bootstrap_build_tags.sh‎

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ source "$SCRIPTDIR/common.sh"
88

99
# bootstrap stevedore with 1 change log
1010
fromager \
11-
--log-file="$OUTDIR/bootstrap.log" \
11+
--log-file="$OUTDIR/bootstrap1.log" \
1212
--error-log-file="$OUTDIR/fromager-errors.log" \
1313
--sdists-repo="$OUTDIR/sdists-repo" \
1414
--wheels-repo="$OUTDIR/wheels-repo" \
@@ -20,8 +20,6 @@ EXPECTED_FILES="
2020
$OUTDIR/wheels-repo/downloads/stevedore-5.2.0-1*.whl
2121
"
2222

23-
start_local_wheel_server
24-
2523
pass=true
2624
for pattern in $EXPECTED_FILES; do
2725
if [ ! -f "${pattern}" ]; then
@@ -32,26 +30,64 @@ done
3230

3331
$pass
3432

33+
# bootstrap stevedore again with 1 change log without a file in the downloads
34+
# cache but with a file in the cache package server. Should not result in a
35+
# build because the file exists in the cache server directory.
36+
37+
find "$OUTDIR/wheels-repo"
38+
# replace the symlink in the index dir with an actual file so it can be
39+
# downloaded, and remove the wheels from the disk cache so we know we get the
40+
# file from the web server
41+
rm -f $OUTDIR/wheels-repo/simple/stevedore/*.whl
42+
mv $OUTDIR/wheels-repo/downloads/stevedore*.whl $OUTDIR/wheels-repo/simple/stevedore/
43+
rm -f $OUTDIR/wheels-repo/build/*.whl
44+
rm -f $OUTDIR/wheels-repo/downloads/*.whl
45+
find "$OUTDIR/wheels-repo"
46+
start_local_wheel_server
47+
48+
LOGFILE="$OUTDIR/bootstrap2.log"
49+
fromager \
50+
--log-file="$LOGFILE" \
51+
--error-log-file="$OUTDIR/fromager-errors.log" \
52+
--sdists-repo="$OUTDIR/sdists-repo" \
53+
--wheels-repo="$OUTDIR/wheels-repo" \
54+
--work-dir="$OUTDIR/work-dir" \
55+
--settings-file="$SCRIPTDIR/bootstrap_settings.yaml" \
56+
bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
57+
58+
if ! grep -q "stevedore: found built wheel on cache server" "$LOGFILE"; then
59+
echo "FAIL: Did not find log message found built wheel on cache server in $LOGFILE" 1>&2
60+
pass=false
61+
fi
62+
63+
$pass
64+
3565
# bootstrap stevedore again with 1 change log. Should not result in a build
66+
# because the file exists in the cache server directory.
67+
68+
find "$OUTDIR/wheels-repo"
69+
70+
LOGFILE="$OUTDIR/bootstrap3.log"
3671
fromager \
37-
--log-file="$OUTDIR/bootstrap.log" \
72+
--log-file="$LOGFILE" \
3873
--error-log-file="$OUTDIR/fromager-errors.log" \
3974
--sdists-repo="$OUTDIR/sdists-repo" \
4075
--wheels-repo="$OUTDIR/wheels-repo" \
4176
--work-dir="$OUTDIR/work-dir" \
4277
--settings-file="$SCRIPTDIR/bootstrap_settings.yaml" \
4378
bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
4479

45-
if ! grep -q "stevedore: found built wheel on cache server" "$OUTDIR/bootstrap.log"; then
46-
echo "FAIL: Did not find log message found built wheel on cache server in $OUTDIR/bootstrap.log" 1>&2
80+
if ! grep -q "stevedore: found existing wheel " "$LOGFILE"; then
81+
echo "FAIL: Did not find log message found existing wheel in $LOGFILE" 1>&2
4782
pass=false
4883
fi
4984

5085
$pass
5186

5287
# bootstrap stevedore with 2 changelog. should result in a build instead of being skipped
88+
LOGFILE="$OUTDIR/bootstrap4.log"
5389
fromager \
54-
--log-file="$OUTDIR/bootstrap_build_tags.log" \
90+
--log-file="$LOGFILE" \
5591
--error-log-file="$OUTDIR/fromager-errors.log" \
5692
--sdists-repo="$OUTDIR/sdists-repo" \
5793
--wheels-repo="$OUTDIR/wheels-repo" \
@@ -72,8 +108,8 @@ for pattern in $EXPECTED_FILES; do
72108
fi
73109
done
74110

75-
if ! grep -q "added extra metadata and build tag" "$OUTDIR/bootstrap_build_tags.log"; then
76-
echo "Did not find message indicating builds would be skipped" 1>&2
111+
if ! grep -q "added extra metadata and build tag" "$LOGFILE"; then
112+
echo "Did not find message indicating builds would be skipped in $LOGFILE" 1>&2
77113
pass=false
78114
fi
79115

‎e2e/test_bootstrap_cache.sh‎

Lines changed: 44 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -7,96 +7,89 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
77
source "$SCRIPTDIR/common.sh"
88
pass=true
99

10+
DIST=setuptools
11+
VER=78.1.0
12+
13+
################################################################################
1014
# run fromager once to build wheels that can be used by a local wheel server
15+
1116
fromager \
17+
--debug \
1218
--log-file="$OUTDIR/bootstrap.log" \
1319
--error-log-file="$OUTDIR/fromager-errors.log" \
1420
--sdists-repo="$OUTDIR/sdists-repo" \
1521
--wheels-repo="$OUTDIR/wheels-repo" \
1622
--work-dir="$OUTDIR/work-dir" \
1723
--settings-dir="$SCRIPTDIR/changelog_settings" \
18-
bootstrap 'stevedore==5.2.0'
24+
bootstrap "$DIST==$VER"
1925

20-
start_local_wheel_server
21-
rm -rf "$OUTDIR/sdists-repo"
26+
################################################################################
27+
# run fromager again to verify that we pick up the existing wheels in the output directory
28+
29+
# Remove build work-dir and wheel server dir, but not the sdist or wheel repos
2230
rm -rf "$OUTDIR/work-dir"
31+
rm -rf "$OUTDIR/wheels-repo/simple"
2332
rm "$OUTDIR/bootstrap.log"
2433

25-
# run fromager with the cache wheel server pointing to the local wheel server
2634
fromager \
35+
--debug \
2736
--log-file="$OUTDIR/bootstrap.log" \
2837
--error-log-file="$OUTDIR/fromager-errors.log" \
2938
--sdists-repo="$OUTDIR/sdists-repo" \
3039
--wheels-repo="$OUTDIR/wheels-repo" \
3140
--work-dir="$OUTDIR/work-dir" \
3241
--settings-dir="$SCRIPTDIR/changelog_settings" \
33-
--no-cleanup \
34-
bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
42+
bootstrap --cache-wheel-server-url="https://pypi.org/simple" "$DIST==$VER"
3543

3644
EXPECTED_LOG_MESSAGES=(
37-
"loading build sdist dependencies from build-sdist-requirements.txt"
38-
"loading build backend dependencies from build-backend-requirements.txt"
39-
"loading build system dependencies from build-system-requirements.txt"
45+
"$DIST: looking for existing wheel for version $VER with build tag () in"
46+
"$DIST: found existing wheel"
4047
)
41-
4248
for pattern in "${EXPECTED_LOG_MESSAGES[@]}"; do
43-
echo $pattern
44-
if ! grep -q "stevedore: $pattern" "$OUTDIR/bootstrap.log"; then
45-
echo "FAIL: Did not find log message stevedore: $pattern in $OUTDIR/bootstrap.log" 1>&2
46-
pass=false
47-
fi
48-
if ! grep -q "pbr: $pattern" "$OUTDIR/bootstrap.log"; then
49-
echo "FAIL: Did not find log message pbr: $pattern in $OUTDIR/bootstrap.log" 1>&2
50-
pass=false
51-
fi
52-
if ! grep -q "setuptools: $pattern" "$OUTDIR/bootstrap.log"; then
53-
echo "FAIL: Did not find log message setuptools: $pattern in $OUTDIR/bootstrap.log" 1>&2
49+
if ! grep -q "$pattern" "$OUTDIR/bootstrap.log"; then
50+
echo "FAIL: Did not find log message $pattern in $OUTDIR/bootstrap.log" 1>&2
5451
pass=false
5552
fi
5653
done
54+
$pass
5755

58-
EXPECTED_FILES="
59-
$OUTDIR/wheels-repo/downloads/setuptools-*.whl
60-
$OUTDIR/wheels-repo/downloads/pbr-*.whl
61-
$OUTDIR/wheels-repo/downloads/stevedore-*.whl
62-
63-
$OUTDIR/work-dir/pbr-*/*-requirements.txt
64-
$OUTDIR/work-dir/setuptools-*/*-requirements.txt
65-
$OUTDIR/work-dir/stevedore-*/*-requirements.txt
66-
67-
$OUTDIR/work-dir/build-order.json
68-
$OUTDIR/work-dir/constraints.txt
69-
"
56+
UNEXPECTED_LOG_MESSAGES=(
57+
"$DIST: checking if wheel was already uploaded to https://pypi.org/simple"
58+
)
7059

71-
for pattern in $EXPECTED_FILES; do
72-
if [ ! -f "${pattern}" ]; then
73-
echo "Did not find $pattern" 1>&2
60+
for pattern in "${UNEXPECTED_LOG_MESSAGES[@]}"; do
61+
if grep -q "$pattern" "$OUTDIR/bootstrap.log"; then
62+
echo "FAIL: Found log message $pattern in $OUTDIR/bootstrap.log" 1>&2
7463
pass=false
7564
fi
7665
done
77-
7866
$pass
7967

68+
################################################################################
69+
70+
# run fromager with the cache wheel server pointing to the pypi server and
71+
# verify we can pick it up from there
72+
73+
start_local_wheel_server
8074
rm -rf "$OUTDIR/sdists-repo"
8175
rm -rf "$OUTDIR/work-dir"
82-
rm -rf "$OUTDIR/wheels-repo"
76+
rm -rf "$OUTDIR/wheels-repo/build"
77+
rm -rf "$OUTDIR/wheels-repo/download"
8378
rm "$OUTDIR/bootstrap.log"
8479

85-
# run fromager with the cache wheel server pointing to the pypi server
8680
fromager \
81+
--debug \
8782
--log-file="$OUTDIR/bootstrap.log" \
8883
--error-log-file="$OUTDIR/fromager-errors.log" \
8984
--sdists-repo="$OUTDIR/sdists-repo" \
9085
--wheels-repo="$OUTDIR/wheels-repo" \
9186
--work-dir="$OUTDIR/work-dir" \
9287
--settings-dir="$SCRIPTDIR/changelog_settings" \
9388
--no-cleanup \
94-
bootstrap --cache-wheel-server-url="https://pypi.org/simple" 'stevedore==5.2.0'
89+
bootstrap --cache-wheel-server-url="https://pypi.org/simple" "$DIST==$VER"
9590

9691
EXPECTED_FILES="
9792
$OUTDIR/wheels-repo/downloads/setuptools-*.whl
98-
$OUTDIR/wheels-repo/downloads/pbr-*.whl
99-
$OUTDIR/wheels-repo/downloads/stevedore-*.whl
10093
10194
$OUTDIR/work-dir/build-order.json
10295
$OUTDIR/work-dir/constraints.txt
@@ -111,18 +104,16 @@ done
111104

112105
$pass
113106

114-
for pattern in "${EXPECTED_LOG_MESSAGES[@]}"; do
107+
UNEXPECTED_LOG_MESSAGES=(
108+
"$DIST: loading build sdist dependencies from build-sdist-requirements.txt"
109+
"$DIST: loading build backend dependencies from build-backend-requirements.txt"
110+
"$DIST: loading build system dependencies from build-system-requirements.txt"
111+
)
112+
113+
for pattern in "${UNEXPECTED_LOG_MESSAGES[@]}"; do
115114
echo $pattern
116-
if grep -q "stevedore: $pattern" "$OUTDIR/bootstrap.log"; then
117-
echo "FAIL: found log message stevedore: $pattern in $OUTDIR/bootstrap.log" 1>&2
118-
pass=false
119-
fi
120-
if grep -q "pbr: $pattern" "$OUTDIR/bootstrap.log"; then
121-
echo "FAIL: found log message pbr: $pattern in $OUTDIR/bootstrap.log" 1>&2
122-
pass=false
123-
fi
124-
if grep -q "setuptools: $pattern" "$OUTDIR/bootstrap.log"; then
125-
echo "FAIL: found log message setuptools: $pattern in $OUTDIR/bootstrap.log" 1>&2
115+
if grep -q "$pattern" "$OUTDIR/bootstrap.log"; then
116+
echo "FAIL: found log message $pattern in $OUTDIR/bootstrap.log" 1>&2
126117
pass=false
127118
fi
128119
done

0 commit comments

Comments
 (0)